feat(heads): auto-apply observability + on by default (#114 auto-apply B)
Auto-apply is now ON by default (operator-asked: opt-OUT, not opt-in) — migration 0059 + model default flipped. The support (>=30) + measured-precision gates keep it safe and every auto-tag is reversible. Observability so the operator can tune from real data: - MISFIRE = an auto-applied (source='head_auto') tag the operator later removes. UNDER-FIRE = a tag with a head the operator adds by hand (the head missed it). Both captured at correction time in TagService.add_to_image/remove_from_image (source is lost on delete) into durable per-tag counters (head_metric), keyed by tag so they survive head retrain/prune. - Daily snapshot_head_metrics writes a per-concept time-series point (head_metrics_snapshot): auto-applied volume + cumulative misfires/under-fires + head quality; 180-day retention; daily beat. - GET /api/heads/metrics: per-concept current counts + realized misfire rate + head quality, plus the snapshot time-series — the report to tune the precision target + support floor. Migration 0060. Tests: misfire/under-fire counting (and the negatives — manual removal isn't a misfire, headless manual add isn't an under-fire), snapshot time-series, metrics API. What's the autofire threshold? There's no single number — each graduated head derives its OWN probability cutoff from its PR curve: the operating point that holds precision >= head_auto_apply_precision (0.97) at max recall. The global knobs are that target + the >=30 support floor. NEXT (slice 3): UI — enable toggle, dry-run preview, per-concept trends. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
@@ -49,7 +49,7 @@ def upgrade() -> None:
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"head_auto_apply_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.false(),
|
||||
server_default=sa.true(), # opt-out: on by default (operator-asked)
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""head_metric + head_metrics_snapshot: auto-apply observability (#114)
|
||||
|
||||
Running misfire/under-fire counters per concept (captured at correction time,
|
||||
since image_tag.source is lost on delete) + a daily per-concept time-series so
|
||||
the operator can tune the precision target + support floor from real data.
|
||||
|
||||
Revision ID: 0060
|
||||
Revises: 0059
|
||||
Create Date: 2026-06-29
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0060"
|
||||
down_revision: Union[str, None] = "0059"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"head_metric",
|
||||
sa.Column(
|
||||
"tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
||||
),
|
||||
sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"head_metrics_snapshot",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"),
|
||||
),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"snapshot_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("n_auto_applied", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("ap", sa.Float(), nullable=True),
|
||||
sa.Column("precision_cv", sa.Float(), nullable=True),
|
||||
sa.Column("recall", sa.Float(), nullable=True),
|
||||
sa.Column("n_pos", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_head_metrics_snapshot_tag_id", "head_metrics_snapshot", ["tag_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_head_metrics_snapshot_snapshot_at", "head_metrics_snapshot",
|
||||
["snapshot_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_head_metrics_snapshot_snapshot_at", table_name="head_metrics_snapshot"
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_head_metrics_snapshot_tag_id", table_name="head_metrics_snapshot"
|
||||
)
|
||||
op.drop_table("head_metrics_snapshot")
|
||||
op.drop_table("head_metric")
|
||||
Reference in New Issue
Block a user