bvandeusen
666b3a2ec8
Consolidate duplication accrued across the ML tagging + settings backend, behavior-preserving (over-DRY guard applied — the three auto-apply sweep BODIES stay separate; only their shared inner helpers are extracted). - _sigmoid / _conflict_scores / _insert_presentation_review (heads.py): the score→prob transform (6 inlined sites), the presentation conflict signal (2 sites), and the ring-loud PresentationReview insert (2 sites, single- sourced so the mode column can't drift on the shared composite PK). - _applied_or_rejected (training_data.py): the per-tag "applied ∪ rejected" skip-set, byte-identical at 3 sweep sites (heads.py x2, tasks/ml.py ccip). - ccip sweep divergence fixes: import ccip._FIGURE_KINDS + training_data._l2norm instead of local copies that silently drift when the canonical changes. - MLSettings.load / .load_sync classmethods (mirror ImportSettings); route all 8 scalar_one singleton reads through them (the session.get None-path stays). - GET serializers for MLSettings + ImportSettings are now table-driven off the same _EDITABLE tuples PATCH writes, so a new field can't be silently absent from GET (the split that historically dropped fields). - AUTO_APPLY_THRESHOLD_MIN/MAX constant single-sources the [0.5,0.999] operating range across the service clamp + the 5 API validators. - test_ml_dry_helpers.py pins _applied_or_rejected + _sigmoid. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NsmJSQxnNxGgtM5Yz4GAqi
60 lines
2.5 KiB
Python
60 lines
2.5 KiB
Python
"""Shared ML helpers extracted in the DRY pass (milestone #161). These pin the
|
||
single sources the auto-apply sweeps now trust, so a future edit can't silently
|
||
drift them: `_applied_or_rejected` is the skip-set used by auto_apply_sweep,
|
||
system_tag_auto_apply_sweep (heads.py) and scheduled_ccip_auto_apply (tasks/ml.py);
|
||
`_sigmoid` is the head score→prob transform used at every scoring site."""
|
||
import pytest
|
||
|
||
from backend.app.models import ImageRecord, Tag, TagKind, TagSuggestionRejection
|
||
from backend.app.models.tag import image_tag
|
||
from backend.app.services.ml.training_data import _applied_or_rejected
|
||
|
||
|
||
def test_sigmoid_matches_naive_form():
|
||
import numpy as np
|
||
|
||
from backend.app.services.ml.heads import _sigmoid
|
||
|
||
z = np.array([-3.0, -0.5, 0.0, 1.5, 12.0], dtype=np.float32)
|
||
assert np.allclose(_sigmoid(z, np), 1.0 / (1.0 + np.exp(-z)))
|
||
assert float(_sigmoid(np.array([0.0]), np)[0]) == pytest.approx(0.5)
|
||
|
||
|
||
@pytest.mark.integration
|
||
def test_applied_or_rejected_unions_applied_any_source_and_rejected(db_sync):
|
||
a = Tag(name="dry-helper-a", kind=TagKind.general)
|
||
b = Tag(name="dry-helper-b", kind=TagKind.general)
|
||
db_sync.add_all([a, b])
|
||
db_sync.flush()
|
||
|
||
imgs = []
|
||
for i in range(5):
|
||
img = ImageRecord(
|
||
path=f"/images/dryhelp{i}.jpg", sha256=f"{i:064d}", size_bytes=1,
|
||
mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
|
||
integrity_status="unknown", siglip_embedding=[0.0] * 1152,
|
||
)
|
||
db_sync.add(img)
|
||
imgs.append(img)
|
||
db_sync.flush()
|
||
|
||
# tag a: applied manually (img0), applied by an AUTO source (img1), rejected (img2).
|
||
db_sync.execute(image_tag.insert().values(
|
||
image_record_id=imgs[0].id, tag_id=a.id, source="manual"))
|
||
db_sync.execute(image_tag.insert().values(
|
||
image_record_id=imgs[1].id, tag_id=a.id, source="head_auto"))
|
||
db_sync.add(TagSuggestionRejection(image_record_id=imgs[2].id, tag_id=a.id))
|
||
# tag b: applied to img3 only.
|
||
db_sync.execute(image_tag.insert().values(
|
||
image_record_id=imgs[3].id, tag_id=b.id, source="manual"))
|
||
db_sync.flush()
|
||
|
||
skip = _applied_or_rejected(db_sync, [a.id, b.id])
|
||
|
||
# Applied-under-ANY-source (manual + head_auto) ∪ rejected, kept per-tag; the
|
||
# untouched image (img4) appears under neither tag.
|
||
assert skip[a.id] == {imgs[0].id, imgs[1].id, imgs[2].id}
|
||
assert skip[b.id] == {imgs[3].id}
|
||
assert imgs[4].id not in skip[a.id]
|
||
assert imgs[4].id not in skip[b.id]
|