feat(heads): earned auto-apply — sweep mechanism, off by default (#114 auto-apply A)
Graduated heads can now apply their tag without a human — gated so it's safe:
- FIRING GATE: a head fires only when the master switch (head_auto_apply_enabled,
default OFF) is on AND it has >= head_auto_apply_min_positives (default 30)
clean labels. A precise-looking but under-supported low-N head can't spray tags.
- auto_apply_sweep (heads.py): streams every embedded image in chunks, scores
against the eligible heads (numpy, no sklearn), applies each head's tag where
score >= its auto_apply_threshold and the tag isn't already applied/rejected,
with source='head_auto' (distinguishable + reversible). dry_run counts only.
- HeadAutoApplyRun (migration 0059) tracks each sweep / preview; apply_head_tags
task (ml queue) + scheduled_apply_head_tags daily beat (no-op unless enabled)
+ recovery sweep + retention(20).
- API: POST /api/heads/auto-apply {dry_run} (202 / 409 running / 400 disabled),
GET /api/heads/auto-apply (recent runs + per-concept report). Settings
head_auto_apply_enabled + min_positives via /api/ml/settings.
Tests: sweep applies above threshold, dry-run writes nothing, skips under-
supported + ungraduated heads; API disabled/dry-run/conflict guards.
NEXT (slice 2): the observability the operator asked for — per-concept misfire
(auto-applied-then-removed) + under-fire tracking, time-series snapshots, and a
reporting API to tune. Slice 3: the UI (enable, preview, trends).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
@@ -0,0 +1,70 @@
|
|||||||
|
"""head_auto_apply_run + earned-auto-apply settings (#114)
|
||||||
|
|
||||||
|
A graduated head can apply its tag without a human, gated by a master switch +
|
||||||
|
a support floor. head_auto_apply_run tracks each sweep / dry-run preview.
|
||||||
|
|
||||||
|
Revision ID: 0059
|
||||||
|
Revises: 0058
|
||||||
|
Create Date: 2026-06-29
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
revision: str = "0059"
|
||||||
|
down_revision: Union[str, None] = "0058"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"head_auto_apply_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"dry_run", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||||
|
),
|
||||||
|
sa.Column("params", JSONB(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), nullable=False,
|
||||||
|
server_default="running",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True), nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("n_applied", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("report", JSONB(), nullable=True),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_head_auto_apply_run_status", "head_auto_apply_run", ["status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"head_auto_apply_enabled", sa.Boolean(), nullable=False,
|
||||||
|
server_default=sa.false(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"head_auto_apply_min_positives", sa.Integer(), nullable=False,
|
||||||
|
server_default="30",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("ml_settings", "head_auto_apply_min_positives")
|
||||||
|
op.drop_column("ml_settings", "head_auto_apply_enabled")
|
||||||
|
op.drop_index(
|
||||||
|
"ix_head_auto_apply_run_status", table_name="head_auto_apply_run"
|
||||||
|
)
|
||||||
|
op.drop_table("head_auto_apply_run")
|
||||||
@@ -12,8 +12,14 @@ from quart import Blueprint, jsonify, request
|
|||||||
from sqlalchemy import desc, func, select
|
from sqlalchemy import desc, func, select
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import HeadTrainingRun, Tag, TagHead
|
from ..models import HeadAutoApplyRun, HeadTrainingRun, Tag, TagHead
|
||||||
from ..services.ml.heads import HeadTrainingAlreadyRunning, start_head_training_run
|
from ..services.ml.heads import (
|
||||||
|
HeadAutoApplyAlreadyRunning,
|
||||||
|
HeadAutoApplyDisabled,
|
||||||
|
HeadTrainingAlreadyRunning,
|
||||||
|
start_head_auto_apply_run,
|
||||||
|
start_head_training_run,
|
||||||
|
)
|
||||||
|
|
||||||
heads_bp = Blueprint("heads", __name__, url_prefix="/api/heads")
|
heads_bp = Blueprint("heads", __name__, url_prefix="/api/heads")
|
||||||
|
|
||||||
@@ -116,3 +122,62 @@ async def status():
|
|||||||
"runs": [_serialize_run(r) for r in runs],
|
"runs": [_serialize_run(r) for r in runs],
|
||||||
"heads": heads,
|
"heads": heads,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_apply_run(run: HeadAutoApplyRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": run.id,
|
||||||
|
"dry_run": run.dry_run,
|
||||||
|
"status": run.status,
|
||||||
|
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||||
|
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||||
|
"n_applied": run.n_applied,
|
||||||
|
"report": run.report,
|
||||||
|
"error": run.error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@heads_bp.route("/auto-apply", methods=["POST"])
|
||||||
|
async def auto_apply():
|
||||||
|
"""Trigger an earned-auto-apply sweep. {dry_run:true} previews (writes
|
||||||
|
nothing); a real sweep needs head_auto_apply_enabled on."""
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
params = {"dry_run": bool(body.get("dry_run", False))}
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
run_id = await session.run_sync(
|
||||||
|
lambda s: start_head_auto_apply_run(s, params)
|
||||||
|
)
|
||||||
|
except HeadAutoApplyAlreadyRunning as running:
|
||||||
|
return jsonify({
|
||||||
|
"error": "auto_apply_already_running",
|
||||||
|
"running_id": int(running.args[0]),
|
||||||
|
}), 409
|
||||||
|
except HeadAutoApplyDisabled:
|
||||||
|
return jsonify({"error": "auto_apply_disabled"}), 400
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"run_id": run_id, "status": "running"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@heads_bp.route("/auto-apply", methods=["GET"])
|
||||||
|
async def auto_apply_status():
|
||||||
|
async with get_session() as session:
|
||||||
|
running = (
|
||||||
|
await session.execute(
|
||||||
|
select(HeadAutoApplyRun.id)
|
||||||
|
.where(HeadAutoApplyRun.status == "running")
|
||||||
|
.order_by(HeadAutoApplyRun.id.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
runs = (
|
||||||
|
await session.execute(
|
||||||
|
select(HeadAutoApplyRun)
|
||||||
|
.order_by(HeadAutoApplyRun.id.desc())
|
||||||
|
.limit(10)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return jsonify({
|
||||||
|
"running_id": running,
|
||||||
|
"runs": [_serialize_apply_run(r) for r in runs],
|
||||||
|
})
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ _EDITABLE = (
|
|||||||
"video_min_tag_frames",
|
"video_min_tag_frames",
|
||||||
"head_min_positives",
|
"head_min_positives",
|
||||||
"head_auto_apply_precision",
|
"head_auto_apply_precision",
|
||||||
|
"head_auto_apply_enabled",
|
||||||
|
"head_auto_apply_min_positives",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -44,6 +46,8 @@ async def get_settings():
|
|||||||
"embedder_model_version": s.embedder_model_version,
|
"embedder_model_version": s.embedder_model_version,
|
||||||
"head_min_positives": s.head_min_positives,
|
"head_min_positives": s.head_min_positives,
|
||||||
"head_auto_apply_precision": s.head_auto_apply_precision,
|
"head_auto_apply_precision": s.head_auto_apply_precision,
|
||||||
|
"head_auto_apply_enabled": s.head_auto_apply_enabled,
|
||||||
|
"head_auto_apply_min_positives": s.head_auto_apply_min_positives,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -109,6 +113,8 @@ def _validate(p: dict) -> str | None:
|
|||||||
return "head_min_positives must be >= 1"
|
return "head_min_positives must be >= 1"
|
||||||
if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
|
if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
|
||||||
return "head_auto_apply_precision must be between 0.5 and 0.999"
|
return "head_auto_apply_precision must be between 0.5 and 0.999"
|
||||||
|
if int(p["head_auto_apply_min_positives"]) < 1:
|
||||||
|
return "head_auto_apply_min_positives must be >= 1"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -113,6 +113,10 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
||||||
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
||||||
},
|
},
|
||||||
|
"apply-head-tags-daily": {
|
||||||
|
"task": "backend.app.tasks.ml.scheduled_apply_head_tags",
|
||||||
|
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
||||||
|
},
|
||||||
"integrity-verify-weekly": {
|
"integrity-verify-weekly": {
|
||||||
"task": "backend.app.tasks.maintenance.verify_integrity",
|
"task": "backend.app.tasks.maintenance.verify_integrity",
|
||||||
"schedule": 604800.0, # weekly
|
"schedule": 604800.0, # weekly
|
||||||
@@ -168,6 +172,10 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.recover_stalled_head_training_runs",
|
"task": "backend.app.tasks.maintenance.recover_stalled_head_training_runs",
|
||||||
"schedule": 300.0,
|
"schedule": 300.0,
|
||||||
},
|
},
|
||||||
|
"recover-stalled-head-auto-apply-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs",
|
||||||
|
"schedule": 300.0,
|
||||||
|
},
|
||||||
"recover-stalled-import-batches": {
|
"recover-stalled-import-batches": {
|
||||||
"task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
|
"task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
|
||||||
"schedule": 300.0,
|
"schedule": 300.0,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from .base import Base
|
|||||||
from .credential import Credential
|
from .credential import Credential
|
||||||
from .download_event import DownloadEvent
|
from .download_event import DownloadEvent
|
||||||
from .external_link import ExternalLink
|
from .external_link import ExternalLink
|
||||||
|
from .head_auto_apply_run import HeadAutoApplyRun
|
||||||
from .head_training_run import HeadTrainingRun
|
from .head_training_run import HeadTrainingRun
|
||||||
from .image_prediction import ImagePrediction
|
from .image_prediction import ImagePrediction
|
||||||
from .image_provenance import ImageProvenance
|
from .image_provenance import ImageProvenance
|
||||||
@@ -67,6 +68,7 @@ __all__ = [
|
|||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
"LibraryAuditRun",
|
"LibraryAuditRun",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
|
"HeadAutoApplyRun",
|
||||||
"HeadTrainingRun",
|
"HeadTrainingRun",
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
"TagAllowlist",
|
"TagAllowlist",
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""HeadAutoApplyRun — persisted lifecycle of an earned-auto-apply sweep (#114).
|
||||||
|
|
||||||
|
A graduated head can apply its tag to images it scores above the head's
|
||||||
|
auto-apply threshold, without a human. This row tracks one such sweep (or a
|
||||||
|
dry-run PREVIEW of it) so the result survives navigation and the admin card can
|
||||||
|
show what fired / what would fire. Mirrors HeadTrainingRun. State machine:
|
||||||
|
running → ready / error. The `report` JSONB holds per-concept counts
|
||||||
|
(applied / projected / scanned).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class HeadAutoApplyRun(Base):
|
||||||
|
__tablename__ = "head_auto_apply_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
# dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing
|
||||||
|
# (preview/apply parity, rule 93).
|
||||||
|
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True
|
||||||
|
)
|
||||||
|
# running | ready | error
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
# Total tags applied across all heads this sweep (0 for a clean dry-run).
|
||||||
|
n_applied: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
# Per-concept breakdown: [{tag_id, name, applied, scanned, threshold}, ...].
|
||||||
|
report: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
@@ -2,7 +2,15 @@
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import CheckConstraint, DateTime, Float, Integer, String, func
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
CheckConstraint,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -66,6 +74,17 @@ class MLSettings(Base):
|
|||||||
head_auto_apply_precision: Mapped[float] = mapped_column(
|
head_auto_apply_precision: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.97
|
Float, nullable=False, default=0.97
|
||||||
)
|
)
|
||||||
|
# Earned auto-apply (#114). A graduated head fires (tags images without a
|
||||||
|
# human) ONLY when this master switch is on AND the head has at least
|
||||||
|
# head_auto_apply_min_positives clean labels — so a precise-looking but
|
||||||
|
# under-supported low-N head can't spray tags across the library. Off by
|
||||||
|
# default; the operator enables after previewing. Operator-tunable.
|
||||||
|
head_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=False
|
||||||
|
)
|
||||||
|
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=30
|
||||||
|
)
|
||||||
tagger_model_version: Mapped[str] = mapped_column(
|
tagger_model_version: Mapped[str] = mapped_column(
|
||||||
String(128), nullable=False, default="camie-tagger-v2"
|
String(128), nullable=False, default="camie-tagger-v2"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -26,12 +26,14 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ...models import (
|
from ...models import (
|
||||||
|
HeadAutoApplyRun,
|
||||||
HeadTrainingRun,
|
HeadTrainingRun,
|
||||||
ImageRecord,
|
ImageRecord,
|
||||||
MLSettings,
|
MLSettings,
|
||||||
Tag,
|
Tag,
|
||||||
TagHead,
|
TagHead,
|
||||||
TagKind,
|
TagKind,
|
||||||
|
TagSuggestionRejection,
|
||||||
)
|
)
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
from .tag_eval import (
|
from .tag_eval import (
|
||||||
@@ -328,3 +330,138 @@ async def _settings_async(session: AsyncSession) -> MLSettings:
|
|||||||
return (
|
return (
|
||||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Earned auto-apply (sync, ml worker) ---------------------------------
|
||||||
|
# A graduated head can apply its tag to images it scores above the head's
|
||||||
|
# auto_apply_threshold, without a human. Gated by a master switch + a support
|
||||||
|
# floor so a precise-looking but under-supported head can't spray tags.
|
||||||
|
|
||||||
|
_AUTO_APPLY_CHUNK = 5000
|
||||||
|
|
||||||
|
|
||||||
|
class HeadAutoApplyAlreadyRunning(Exception):
|
||||||
|
"""Raised when an auto-apply sweep is already in flight."""
|
||||||
|
|
||||||
|
|
||||||
|
class HeadAutoApplyDisabled(Exception):
|
||||||
|
"""Raised when a real (non-dry-run) sweep is requested but the master
|
||||||
|
switch (head_auto_apply_enabled) is off."""
|
||||||
|
|
||||||
|
|
||||||
|
def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int:
|
||||||
|
"""Create a HeadAutoApplyRun + dispatch the ml-queue sweep. dry_run previews
|
||||||
|
(writes nothing); a real sweep needs the master switch on. One run at a time."""
|
||||||
|
dry_run = bool((params or {}).get("dry_run", False))
|
||||||
|
existing = session.execute(
|
||||||
|
select(HeadAutoApplyRun.id).where(HeadAutoApplyRun.status == "running")
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
raise HeadAutoApplyAlreadyRunning(existing)
|
||||||
|
if not dry_run and not _settings(session).head_auto_apply_enabled:
|
||||||
|
raise HeadAutoApplyDisabled()
|
||||||
|
run = HeadAutoApplyRun(
|
||||||
|
dry_run=dry_run, params={"dry_run": dry_run}, status="running",
|
||||||
|
last_progress_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
session.add(run)
|
||||||
|
session.flush()
|
||||||
|
run_id = run.id
|
||||||
|
from ...tasks.ml import apply_head_tags as _task
|
||||||
|
_task.delay(run_id)
|
||||||
|
return run_id
|
||||||
|
|
||||||
|
|
||||||
|
def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
||||||
|
"""Eligible heads to fire: graduated (auto_apply_threshold set), enough
|
||||||
|
support, current embedding. Returns the row list (tag_id/name/weights/...)."""
|
||||||
|
return session.execute(
|
||||||
|
select(
|
||||||
|
TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
|
||||||
|
TagHead.auto_apply_threshold,
|
||||||
|
)
|
||||||
|
.join(Tag, Tag.id == TagHead.tag_id)
|
||||||
|
.where(TagHead.embedding_version == embedding_version)
|
||||||
|
.where(TagHead.auto_apply_threshold.is_not(None))
|
||||||
|
.where(TagHead.n_pos >= min_pos)
|
||||||
|
).all()
|
||||||
|
|
||||||
|
|
||||||
|
def auto_apply_sweep(
|
||||||
|
session: Session, run: HeadAutoApplyRun, dry_run: bool
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Score every embedded image against the eligible heads and apply (or, for
|
||||||
|
dry_run, just count) each head's tag where score >= its auto_apply_threshold
|
||||||
|
and the tag isn't already applied or rejected on that image. Streams
|
||||||
|
embeddings in chunks; commits per chunk on a real run. Returns
|
||||||
|
{n_applied, concepts:[{tag_id,name,applied,scanned,threshold}]}."""
|
||||||
|
import numpy as np
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
|
settings = _settings(session)
|
||||||
|
rows = _auto_apply_heads(
|
||||||
|
session, settings.embedder_model_version,
|
||||||
|
settings.head_auto_apply_min_positives,
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
return {"n_applied": 0, "concepts": []}
|
||||||
|
|
||||||
|
W = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in rows])
|
||||||
|
b = np.asarray([r.bias for r in rows], dtype=np.float32)
|
||||||
|
thr = np.asarray([r.auto_apply_threshold for r in rows], dtype=np.float32)
|
||||||
|
tag_ids = [r.tag_id for r in rows]
|
||||||
|
names = [r.name for r in rows]
|
||||||
|
|
||||||
|
# Skip images that already carry, or have rejected, each tag.
|
||||||
|
skip = {tid: set() for tid in tag_ids}
|
||||||
|
for tid in tag_ids:
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
|
||||||
|
):
|
||||||
|
skip[tid].add(iid)
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
select(TagSuggestionRejection.image_record_id).where(
|
||||||
|
TagSuggestionRejection.tag_id == tid
|
||||||
|
)
|
||||||
|
):
|
||||||
|
skip[tid].add(iid)
|
||||||
|
|
||||||
|
applied = [0] * len(rows)
|
||||||
|
scanned = 0
|
||||||
|
all_ids = list(session.execute(
|
||||||
|
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None))
|
||||||
|
).scalars())
|
||||||
|
for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK):
|
||||||
|
chunk = all_ids[start:start + _AUTO_APPLY_CHUNK]
|
||||||
|
emb = _load_embeddings(session, chunk)
|
||||||
|
cids = [i for i in chunk if i in emb]
|
||||||
|
if not cids:
|
||||||
|
continue
|
||||||
|
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||||
|
probs = 1.0 / (1.0 + np.exp(-(Xn @ W.T + b))) # (N, H)
|
||||||
|
scanned += len(cids)
|
||||||
|
for h in range(len(rows)):
|
||||||
|
tid = tag_ids[h]
|
||||||
|
for idx in np.where(probs[:, h] >= thr[h])[0]:
|
||||||
|
iid = cids[int(idx)]
|
||||||
|
if iid in skip[tid]:
|
||||||
|
continue
|
||||||
|
skip[tid].add(iid)
|
||||||
|
applied[h] += 1
|
||||||
|
if not dry_run:
|
||||||
|
session.execute(
|
||||||
|
pg_insert(image_tag)
|
||||||
|
.values(image_record_id=iid, tag_id=tid, source="head_auto")
|
||||||
|
.on_conflict_do_nothing()
|
||||||
|
)
|
||||||
|
if not dry_run:
|
||||||
|
session.commit()
|
||||||
|
run.last_progress_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
concepts = [
|
||||||
|
{"tag_id": tag_ids[h], "name": names[h], "applied": applied[h],
|
||||||
|
"scanned": scanned, "threshold": float(thr[h])}
|
||||||
|
for h in range(len(rows))
|
||||||
|
]
|
||||||
|
return {"n_applied": sum(applied), "concepts": concepts}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from ..celery_app import celery
|
|||||||
from ..models import (
|
from ..models import (
|
||||||
BackupRun,
|
BackupRun,
|
||||||
DownloadEvent,
|
DownloadEvent,
|
||||||
|
HeadAutoApplyRun,
|
||||||
HeadTrainingRun,
|
HeadTrainingRun,
|
||||||
ImageRecord,
|
ImageRecord,
|
||||||
ImportBatch,
|
ImportBatch,
|
||||||
@@ -101,6 +102,9 @@ TAG_EVAL_KEEP_RUNS = 20
|
|||||||
# head training (#114) has a 60-min soft limit; flag no-progress past 75.
|
# head training (#114) has a 60-min soft limit; flag no-progress past 75.
|
||||||
HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
|
HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
|
||||||
HEAD_TRAINING_KEEP_RUNS = 20
|
HEAD_TRAINING_KEEP_RUNS = 20
|
||||||
|
# head auto-apply (#114) shares the 60-min soft limit; flag past 75.
|
||||||
|
HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES = 75
|
||||||
|
HEAD_AUTO_APPLY_KEEP_RUNS = 20
|
||||||
# Import batches finalize only after every child ImportTask hits a
|
# Import batches finalize only after every child ImportTask hits a
|
||||||
# terminal state. The recovery sweep targets the case where every
|
# terminal state. The recovery sweep targets the case where every
|
||||||
# task is done but the batch never got its closing UPDATE
|
# task is done but the batch never got its closing UPDATE
|
||||||
@@ -800,6 +804,48 @@ def recover_stalled_head_training_runs() -> int:
|
|||||||
return recovered
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs")
|
||||||
|
def recover_stalled_head_auto_apply_runs() -> int:
|
||||||
|
"""Flip stalled HeadAutoApplyRun 'running' rows to 'error' + prune to the
|
||||||
|
last HEAD_AUTO_APPLY_KEEP_RUNS (retention, rule 89). 5-min maintenance lane."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
cutoff = now - timedelta(minutes=HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
update(HeadAutoApplyRun)
|
||||||
|
.where(HeadAutoApplyRun.status == "running")
|
||||||
|
.where(
|
||||||
|
func.coalesce(
|
||||||
|
HeadAutoApplyRun.last_progress_at, HeadAutoApplyRun.started_at
|
||||||
|
)
|
||||||
|
< cutoff
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status="error", finished_at=now,
|
||||||
|
error=(
|
||||||
|
f"stranded by recovery sweep (no progress for "
|
||||||
|
f"{HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES} min)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
keep = session.execute(
|
||||||
|
select(HeadAutoApplyRun.id).order_by(HeadAutoApplyRun.id.desc())
|
||||||
|
.limit(HEAD_AUTO_APPLY_KEEP_RUNS)
|
||||||
|
).scalars().all()
|
||||||
|
if keep:
|
||||||
|
session.execute(
|
||||||
|
delete(HeadAutoApplyRun).where(HeadAutoApplyRun.id.not_in(keep))
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_head_auto_apply_runs: recovered %d rows", recovered
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
|
||||||
def recover_stalled_import_batches() -> int:
|
def recover_stalled_import_batches() -> int:
|
||||||
"""Finalize ImportBatch rows stuck in running past the hard limit
|
"""Finalize ImportBatch rows stuck in running past the hard limit
|
||||||
|
|||||||
@@ -659,3 +659,82 @@ def scheduled_train_heads() -> str:
|
|||||||
run_id = run.id
|
run_id = run.id
|
||||||
train_heads.delay(run_id)
|
train_heads.delay(run_id)
|
||||||
return "dispatched"
|
return "dispatched"
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.ml.apply_head_tags",
|
||||||
|
bind=True,
|
||||||
|
# Scores the whole library against the graduated heads and applies their
|
||||||
|
# tags (or, dry_run, just counts). Streams embeddings in chunks; numpy only,
|
||||||
|
# but ml queue keeps it off the API workers. Commits per chunk.
|
||||||
|
soft_time_limit=3600, time_limit=3900,
|
||||||
|
)
|
||||||
|
def apply_head_tags(self, run_id: int) -> str:
|
||||||
|
"""Run an earned-auto-apply sweep into the persisted HeadAutoApplyRun row."""
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from ..models import HeadAutoApplyRun
|
||||||
|
from ..services.ml.heads import auto_apply_sweep
|
||||||
|
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
run = session.get(HeadAutoApplyRun, run_id)
|
||||||
|
if run is None:
|
||||||
|
return "missing"
|
||||||
|
run.last_progress_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
try:
|
||||||
|
result = auto_apply_sweep(session, run, run.dry_run)
|
||||||
|
except SoftTimeLimitExceeded:
|
||||||
|
run.status = "error"
|
||||||
|
run.error = "timed out"
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
log.exception("apply_head_tags %d failed", run_id)
|
||||||
|
run.status = "error"
|
||||||
|
run.error = str(exc)
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
return "error"
|
||||||
|
run.n_applied = result["n_applied"]
|
||||||
|
run.report = {"concepts": result["concepts"]}
|
||||||
|
run.status = "ready"
|
||||||
|
run.finished_at = datetime.now(UTC)
|
||||||
|
session.commit()
|
||||||
|
return "ready"
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.ml.scheduled_apply_head_tags")
|
||||||
|
def scheduled_apply_head_tags() -> str:
|
||||||
|
"""Daily passive auto-apply sweep (#114) — only when the master switch is on.
|
||||||
|
Skips if a sweep is already in flight. Creates + COMMITS the run before
|
||||||
|
dispatching so the worker always finds it."""
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import select as sa_select
|
||||||
|
|
||||||
|
from ..models import HeadAutoApplyRun, MLSettings
|
||||||
|
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
enabled = session.execute(
|
||||||
|
sa_select(MLSettings.head_auto_apply_enabled).where(MLSettings.id == 1)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if not enabled:
|
||||||
|
return "disabled"
|
||||||
|
running = session.execute(
|
||||||
|
sa_select(HeadAutoApplyRun.id).where(HeadAutoApplyRun.status == "running")
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if running is not None:
|
||||||
|
return "already running"
|
||||||
|
run = HeadAutoApplyRun(
|
||||||
|
dry_run=False, params={"dry_run": False, "source": "scheduled"},
|
||||||
|
status="running", last_progress_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
session.add(run)
|
||||||
|
session.commit()
|
||||||
|
run_id = run.id
|
||||||
|
apply_head_tags.delay(run_id)
|
||||||
|
return "dispatched"
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
"""Earned auto-apply (#114). The sweep is numpy-only (no scikit-learn), so the
|
||||||
|
apply logic is tested directly via the sync session; the API guards (disabled /
|
||||||
|
dry-run / conflict) via the async client."""
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import (
|
||||||
|
HeadAutoApplyRun,
|
||||||
|
ImageRecord,
|
||||||
|
MLSettings,
|
||||||
|
Tag,
|
||||||
|
TagHead,
|
||||||
|
TagKind,
|
||||||
|
)
|
||||||
|
from backend.app.models.tag import image_tag
|
||||||
|
from backend.app.services.ml.heads import auto_apply_sweep
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _emb(slot: int) -> list[float]:
|
||||||
|
v = [0.0] * 1152
|
||||||
|
v[slot] = 3.0
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
def _img(db, sha: str, emb) -> ImageRecord:
|
||||||
|
img = ImageRecord(
|
||||||
|
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
||||||
|
width=1, height=1, origin="imported_filesystem",
|
||||||
|
integrity_status="unknown", siglip_embedding=emb,
|
||||||
|
)
|
||||||
|
db.add(img)
|
||||||
|
db.flush()
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def _head(db, tag_id: int, slot: int, *, threshold=0.5, n_pos=30):
|
||||||
|
s = db.execute(select(MLSettings).where(MLSettings.id == 1)).scalar_one()
|
||||||
|
w = [0.0] * 1152
|
||||||
|
w[slot] = 1.0
|
||||||
|
db.add(TagHead(
|
||||||
|
tag_id=tag_id, embedding_version=s.embedder_model_version,
|
||||||
|
weights=w, bias=0.0, suggest_threshold=0.5, auto_apply_threshold=threshold,
|
||||||
|
n_pos=n_pos, n_neg=90, ap=0.9, precision_cv=0.98, recall=0.7,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db, dry_run=False) -> HeadAutoApplyRun:
|
||||||
|
run = HeadAutoApplyRun(dry_run=dry_run, params={"dry_run": dry_run}, status="running")
|
||||||
|
db.add(run)
|
||||||
|
db.flush()
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def _applied_source(db, image_id, tag_id):
|
||||||
|
return db.execute(
|
||||||
|
select(image_tag.c.source)
|
||||||
|
.where(image_tag.c.image_record_id == image_id)
|
||||||
|
.where(image_tag.c.tag_id == tag_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sweep_applies_to_matching_image(db_sync):
|
||||||
|
img = _img(db_sync, "a" * 64, _emb(0))
|
||||||
|
tag = Tag(name="autotag", kind=TagKind.general)
|
||||||
|
db_sync.add(tag)
|
||||||
|
db_sync.flush()
|
||||||
|
_head(db_sync, tag.id, 0)
|
||||||
|
run = _run(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
result = auto_apply_sweep(db_sync, run, dry_run=False)
|
||||||
|
assert result["n_applied"] == 1
|
||||||
|
assert _applied_source(db_sync, img.id, tag.id) == "head_auto"
|
||||||
|
|
||||||
|
|
||||||
|
def test_sweep_dry_run_counts_but_writes_nothing(db_sync):
|
||||||
|
img = _img(db_sync, "b" * 64, _emb(0))
|
||||||
|
tag = Tag(name="previewtag", kind=TagKind.general)
|
||||||
|
db_sync.add(tag)
|
||||||
|
db_sync.flush()
|
||||||
|
_head(db_sync, tag.id, 0)
|
||||||
|
run = _run(db_sync, dry_run=True)
|
||||||
|
db_sync.commit()
|
||||||
|
result = auto_apply_sweep(db_sync, run, dry_run=True)
|
||||||
|
assert result["n_applied"] == 1 # it WOULD apply
|
||||||
|
assert _applied_source(db_sync, img.id, tag.id) is None # but wrote nothing
|
||||||
|
|
||||||
|
|
||||||
|
def test_sweep_skips_under_supported_head(db_sync):
|
||||||
|
# n_pos below head_auto_apply_min_positives (default 30) → a precise-looking
|
||||||
|
# but under-supported head never fires.
|
||||||
|
img = _img(db_sync, "c" * 64, _emb(0))
|
||||||
|
tag = Tag(name="weaktag", kind=TagKind.general)
|
||||||
|
db_sync.add(tag)
|
||||||
|
db_sync.flush()
|
||||||
|
_head(db_sync, tag.id, 0, n_pos=5)
|
||||||
|
run = _run(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
result = auto_apply_sweep(db_sync, run, dry_run=False)
|
||||||
|
assert result["n_applied"] == 0
|
||||||
|
assert _applied_source(db_sync, img.id, tag.id) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_sweep_skips_ungraduated_head(db_sync):
|
||||||
|
# auto_apply_threshold is None (head never reached the precision bar).
|
||||||
|
img = _img(db_sync, "d" * 64, _emb(0))
|
||||||
|
tag = Tag(name="nograd", kind=TagKind.general)
|
||||||
|
db_sync.add(tag)
|
||||||
|
db_sync.flush()
|
||||||
|
_head(db_sync, tag.id, 0, threshold=None)
|
||||||
|
run = _run(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
result = auto_apply_sweep(db_sync, run, dry_run=False)
|
||||||
|
assert result["n_applied"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_auto_apply_disabled_blocks_real_run(client, db):
|
||||||
|
# head_auto_apply_enabled defaults False → a real sweep is refused (400).
|
||||||
|
resp = await client.post("/api/heads/auto-apply", json={"dry_run": False})
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["error"] == "auto_apply_disabled"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_auto_apply_dry_run_allowed_when_disabled(client, db, monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.ml.apply_head_tags.delay", lambda *a, **k: None
|
||||||
|
)
|
||||||
|
resp = await client.post("/api/heads/auto-apply", json={"dry_run": True})
|
||||||
|
assert resp.status_code == 202
|
||||||
|
assert (await resp.get_json())["status"] == "running"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_auto_apply_conflict_when_one_running(client, db, monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.ml.apply_head_tags.delay", lambda *a, **k: None
|
||||||
|
)
|
||||||
|
db.add(HeadAutoApplyRun(dry_run=True, params={}, status="running"))
|
||||||
|
await db.flush()
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post("/api/heads/auto-apply", json={"dry_run": True})
|
||||||
|
assert resp.status_code == 409
|
||||||
|
assert (await resp.get_json())["error"] == "auto_apply_already_running"
|
||||||
Reference in New Issue
Block a user