FabledScribe/tests/test_retrieval_migration.py

"""Carrying a floor across a calibration change (#4104).

WHAT THIS PINS

  1. **The percentile is what transfers, not the number.** A floor's content is
     a decision about selectivity; the cosine expressing it is units. So a
     migration that admitted 30% before admits 30% after, whatever the new
     scores look like.
  2. **Nothing is written unless asked twice.** `apply` defaults to False. A
     model change makes every number uncertain at once, which is the worst
     moment to let a statistic move six dials unattended.
  3. **Missing evidence is a refusal, not a guess.** No logged calls, or a
     corpus that re-scores to nothing, returns `migrated: False` with a reason —
     never a floor computed off an empty distribution, which is how an install
     mid-backfill would end up with every bar at zero.
  4. **Every registry surface can be migrated.** A seventh arm that nobody adds
     a re-scorer for is one whose floor silently cannot survive a model change.
"""
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from scribe.services import retrieval_migration as rm
from scribe.services.retrieval_surfaces import SURFACES
from tests.helpers import make_mock_session, tool_doc


def _logs(pairs):
    """Rows as `migrate_floor` reads them: (query, project_id, old score)."""
    return [MagicMock(query=q, project_id=p, best_available_score=s)
            for q, p, s in pairs]


def _session_with(rows):
    session = make_mock_session()
    session.execute.return_value.all.return_value = rows
    return session


def test_every_surface_has_a_rescorer():
    """Otherwise a surface's floor cannot cross a model change at all."""
    assert set(rm._RESCORERS) == set(SURFACES)


def test_the_floor_that_admits_a_fraction_is_an_observed_score():
    scores = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.05]
    # 30% of ten is three; the third-best score is the bar that admits exactly
    # those three. Exact rather than interpolated, so the answer can be checked
    # against the sample it came from.
    assert rm._floor_admitting(scores, 0.3) == 0.7
    assert sum(1 for s in scores if s >= 0.7) == 3


def test_a_surface_that_admitted_nothing_keeps_admitting_nothing():
    """A migration must not quietly reopen an arm the operator had shut."""
    scores = [0.5, 0.4, 0.3]
    assert rm._floor_admitting(scores, 0.0) > max(scores)


@pytest.mark.asyncio
async def test_selectivity_is_preserved_across_a_scale_change():
    """The whole idea, on numbers that move a long way.

    Old scores cluster near 0.7 with a bar at 0.72 admitting two of five. New
    scores sit far lower — a different geometry — and the proposal is the value
    that admits two of five there, not anything resembling 0.72.
    """
    rows = _logs([
        ("q1", 2, 0.80), ("q2", 2, 0.75), ("q3", 2, 0.70),
        ("q4", 2, 0.60), ("q5", 2, 0.50),
    ])
    new = {"q1": 0.42, "q2": 0.38, "q3": 0.31, "q4": 0.22, "q5": 0.10}
    rescore = AsyncMock(side_effect=lambda u, q, p: new[q])

    with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \
         patch.object(rm, "floor_for", AsyncMock(return_value=0.72)), \
         patch.dict(rm._RESCORERS, {"prompt_rule": rescore}):
        out = await rm.migrate_floor(1, "prompt_rule")

    assert out["old_admit_rate"] == 0.4          # 0.80 and 0.75 cleared 0.72
    assert out["proposed_floor"] == 0.38         # admits 0.42 and 0.38 — also two
    assert out["migrated"] is False              # dry run by default


@pytest.mark.asyncio
async def test_a_dry_run_writes_nothing():
    rows = _logs([("q1", None, 0.9), ("q2", None, 0.1)])
    with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \
         patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \
         patch.object(rm, "set_dial", AsyncMock()) as set_dial, \
         patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=0.4)}):
        out = await rm.migrate_floor(1, "prompt_rule")
    set_dial.assert_not_called()
    assert "proposed_floor" in out


@pytest.mark.asyncio
async def test_applying_writes_an_ordinary_tuning_event_with_the_arithmetic():
    """A migrated floor is reviewable and revertible like any other change."""
    rows = _logs([("q1", None, 0.9), ("q2", None, 0.1)])
    with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \
         patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \
         patch.object(rm, "set_dial", AsyncMock(return_value={})) as set_dial, \
         patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=0.4)}):
        out = await rm.migrate_floor(1, "prompt_rule", apply=True)

    assert out["migrated"] is True
    kwargs = set_dial.call_args.kwargs
    reason = kwargs["reason"]
    # The reason has to carry the working, not just the verdict — it is what the
    # operator reads to decide whether to keep the number.
    assert "0.5" in reason and "sampled" in reason
    assert kwargs["actor"] == "model"


@pytest.mark.asyncio
async def test_no_logged_calls_refuses_rather_than_inventing_a_distribution():
    with patch.object(rm, "async_session", MagicMock(return_value=_session_with([]))), \
         patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \
         patch.object(rm, "set_dial", AsyncMock()) as set_dial:
        out = await rm.migrate_floor(1, "prompt_rule", apply=True)
    assert out["migrated"] is False
    assert "no logged calls" in out["why"]
    set_dial.assert_not_called()


@pytest.mark.asyncio
async def test_a_corpus_that_rescores_to_nothing_refuses():
    """The mid-backfill case: every bar would otherwise be set off no data."""
    rows = _logs([("q1", None, 0.9), ("q2", None, 0.8)])
    with patch.object(rm, "async_session", MagicMock(return_value=_session_with(rows))), \
         patch.object(rm, "floor_for", AsyncMock(return_value=0.5)), \
         patch.object(rm, "set_dial", AsyncMock()) as set_dial, \
         patch.dict(rm._RESCORERS, {"prompt_rule": AsyncMock(return_value=None)}):
        out = await rm.migrate_floor(1, "prompt_rule", apply=True)
    assert out["migrated"] is False
    assert "not embedded" in out["why"]
    set_dial.assert_not_called()


@pytest.mark.asyncio
async def test_an_unknown_surface_is_refused():
    with pytest.raises(ValueError):
        await rm.migrate_floor(1, "promptrule")


def test_the_tool_says_it_is_a_starting_point_and_defaults_to_a_dry_run():
    doc = tool_doc("scribe.mcp.tools.retrieval_tuning", "migrate_retrieval_floor")
    assert "DRY RUN BY DEFAULT" in doc
    # Percentile-preserving carries the old floor's wrongness forward faithfully.
    # A reader who misses that will treat a migrated number as a measured one.
    assert "STARTING POINT" in doc.upper()
    assert "stale" in doc