bvandeusen
48c8811d69
Auto-apply is now ON by default (operator-asked: opt-OUT, not opt-in) — migration 0059 + model default flipped. The support (>=30) + measured-precision gates keep it safe and every auto-tag is reversible. Observability so the operator can tune from real data: - MISFIRE = an auto-applied (source='head_auto') tag the operator later removes. UNDER-FIRE = a tag with a head the operator adds by hand (the head missed it). Both captured at correction time in TagService.add_to_image/remove_from_image (source is lost on delete) into durable per-tag counters (head_metric), keyed by tag so they survive head retrain/prune. - Daily snapshot_head_metrics writes a per-concept time-series point (head_metrics_snapshot): auto-applied volume + cumulative misfires/under-fires + head quality; 180-day retention; daily beat. - GET /api/heads/metrics: per-concept current counts + realized misfire rate + head quality, plus the snapshot time-series — the report to tune the precision target + support floor. Migration 0060. Tests: misfire/under-fire counting (and the negatives — manual removal isn't a misfire, headless manual add isn't an under-fire), snapshot time-series, metrics API. What's the autofire threshold? There's no single number — each graduated head derives its OWN probability cutoff from its PR curve: the operating point that holds precision >= head_auto_apply_precision (0.97) at max recall. The global knobs are that target + the >=30 support floor. NEXT (slice 3): UI — enable toggle, dry-run preview, per-concept trends. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
75 lines
2.6 KiB
Python
75 lines
2.6 KiB
Python
"""head_metric + head_metrics_snapshot: auto-apply observability (#114)
|
|
|
|
Running misfire/under-fire counters per concept (captured at correction time,
|
|
since image_tag.source is lost on delete) + a daily per-concept time-series so
|
|
the operator can tune the precision target + support floor from real data.
|
|
|
|
Revision ID: 0060
|
|
Revises: 0059
|
|
Create Date: 2026-06-29
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision: str = "0060"
|
|
down_revision: Union[str, None] = "0059"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"head_metric",
|
|
sa.Column(
|
|
"tag_id", sa.Integer(),
|
|
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
|
),
|
|
sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column(
|
|
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
|
server_default=sa.func.now(),
|
|
),
|
|
)
|
|
|
|
op.create_table(
|
|
"head_metrics_snapshot",
|
|
sa.Column("id", sa.Integer(), primary_key=True),
|
|
sa.Column(
|
|
"tag_id", sa.Integer(),
|
|
sa.ForeignKey("tag.id", ondelete="CASCADE"),
|
|
),
|
|
sa.Column("name", sa.String(length=255), nullable=False),
|
|
sa.Column(
|
|
"snapshot_at", sa.DateTime(timezone=True), nullable=False,
|
|
server_default=sa.func.now(),
|
|
),
|
|
sa.Column("n_auto_applied", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("ap", sa.Float(), nullable=True),
|
|
sa.Column("precision_cv", sa.Float(), nullable=True),
|
|
sa.Column("recall", sa.Float(), nullable=True),
|
|
sa.Column("n_pos", sa.Integer(), nullable=True),
|
|
)
|
|
op.create_index(
|
|
"ix_head_metrics_snapshot_tag_id", "head_metrics_snapshot", ["tag_id"],
|
|
)
|
|
op.create_index(
|
|
"ix_head_metrics_snapshot_snapshot_at", "head_metrics_snapshot",
|
|
["snapshot_at"],
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_index(
|
|
"ix_head_metrics_snapshot_snapshot_at", table_name="head_metrics_snapshot"
|
|
)
|
|
op.drop_index(
|
|
"ix_head_metrics_snapshot_tag_id", table_name="head_metrics_snapshot"
|
|
)
|
|
op.drop_table("head_metrics_snapshot")
|
|
op.drop_table("head_metric")
|