bvandeusen
48c8811d69
Auto-apply is now ON by default (operator-asked: opt-OUT, not opt-in) — migration 0059 + model default flipped. The support (>=30) + measured-precision gates keep it safe and every auto-tag is reversible. Observability so the operator can tune from real data: - MISFIRE = an auto-applied (source='head_auto') tag the operator later removes. UNDER-FIRE = a tag with a head the operator adds by hand (the head missed it). Both captured at correction time in TagService.add_to_image/remove_from_image (source is lost on delete) into durable per-tag counters (head_metric), keyed by tag so they survive head retrain/prune. - Daily snapshot_head_metrics writes a per-concept time-series point (head_metrics_snapshot): auto-applied volume + cumulative misfires/under-fires + head quality; 180-day retention; daily beat. - GET /api/heads/metrics: per-concept current counts + realized misfire rate + head quality, plus the snapshot time-series — the report to tune the precision target + support floor. Migration 0060. Tests: misfire/under-fire counting (and the negatives — manual removal isn't a misfire, headless manual add isn't an under-fire), snapshot time-series, metrics API. What's the autofire threshold? There's no single number — each graduated head derives its OWN probability cutoff from its PR curve: the operating point that holds precision >= head_auto_apply_precision (0.97) at max recall. The global knobs are that target + the >=30 support floor. NEXT (slice 3): UI — enable toggle, dry-run preview, per-concept trends. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""Auto-apply observability (#114): misfire/under-fire counters captured on
|
|
operator corrections, the daily snapshot time-series, and the metrics API."""
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import HeadMetric, HeadMetricsSnapshot, ImageRecord, TagHead, TagKind
|
|
from backend.app.models.tag import image_tag
|
|
from backend.app.services.tag_service import TagService
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
async def _img(db, sha) -> ImageRecord:
|
|
img = ImageRecord(
|
|
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
|
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
|
)
|
|
db.add(img)
|
|
await db.flush()
|
|
return img
|
|
|
|
|
|
def _head(tag_id):
|
|
return TagHead(
|
|
tag_id=tag_id, embedding_version="siglip-test", weights=[0.0] * 1152,
|
|
bias=0.0, suggest_threshold=0.5, auto_apply_threshold=0.6,
|
|
n_pos=30, n_neg=90, ap=0.9, precision_cv=0.95, recall=0.7,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_removing_head_auto_tag_counts_misfire(db):
|
|
img = await _img(db, "a" * 64)
|
|
tag = await TagService(db).find_or_create("misfire", TagKind.general)
|
|
await db.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id, source="head_auto",
|
|
))
|
|
await db.commit()
|
|
await TagService(db).remove_from_image(img.id, tag.id)
|
|
await db.commit()
|
|
m = await db.get(HeadMetric, tag.id)
|
|
assert m is not None and m.n_misfires == 1 and m.n_underfires == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_removing_manual_tag_is_not_a_misfire(db):
|
|
img = await _img(db, "b" * 64)
|
|
tag = await TagService(db).find_or_create("manualrm", TagKind.general)
|
|
await db.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id, source="manual",
|
|
))
|
|
await db.commit()
|
|
await TagService(db).remove_from_image(img.id, tag.id)
|
|
await db.commit()
|
|
assert await db.get(HeadMetric, tag.id) is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_manual_add_with_head_counts_underfire(db):
|
|
img = await _img(db, "c" * 64)
|
|
tag = await TagService(db).find_or_create("underfire", TagKind.general)
|
|
db.add(_head(tag.id))
|
|
await db.commit()
|
|
await TagService(db).add_to_image(img.id, tag.id, source="manual")
|
|
await db.commit()
|
|
m = await db.get(HeadMetric, tag.id)
|
|
assert m is not None and m.n_underfires == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_manual_add_without_head_no_underfire(db):
|
|
img = await _img(db, "d" * 64)
|
|
tag = await TagService(db).find_or_create("nohead", TagKind.general)
|
|
await db.commit()
|
|
await TagService(db).add_to_image(img.id, tag.id, source="manual")
|
|
await db.commit()
|
|
assert await db.get(HeadMetric, tag.id) is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_snapshot_records_timeseries_point(db):
|
|
tag = await TagService(db).find_or_create("snap", TagKind.general)
|
|
db.add(_head(tag.id))
|
|
await db.commit()
|
|
from backend.app.tasks.maintenance import snapshot_head_metrics
|
|
|
|
n = snapshot_head_metrics() # sync task, own session
|
|
assert n >= 1
|
|
snaps = (await db.execute(
|
|
select(HeadMetricsSnapshot).where(HeadMetricsSnapshot.tag_id == tag.id)
|
|
)).scalars().all()
|
|
assert len(snaps) == 1
|
|
assert snaps[0].name == "snap"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_metrics_api_returns_concept(client, db):
|
|
tag = await TagService(db).find_or_create("apimetric", TagKind.general)
|
|
db.add(_head(tag.id))
|
|
await db.commit()
|
|
resp = await client.get("/api/heads/metrics")
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
c = next(x for x in body["concepts"] if x["name"] == "apimetric")
|
|
assert c["auto_apply"] is True
|
|
assert c["n_misfires"] == 0
|
|
assert "snapshots" in body
|