FabledScribe/tests/test_retrieval_tuning.py

"""Moving a floor, and the reason that has to come with it (#4102).

WHY THIS EXISTS

The operator handed the dial to the model: *"the user should be able to touch it
but the model should be the thing handling it 9 times out of 10."* Everything
here guards the half of that sentence people skip — the operator still has to be
able to see what was done on their behalf and disagree with it.

WHAT THIS PINS

  1. **A reason is required, and "" does not count.** The column is NOT NULL,
     which an empty string satisfies; the service is where the requirement is
     real. This is the whole guardrail: a caller who must write down why has to
     have looked, and a caller who writes down something wrong leaves the
     operator a sentence to argue with. A number that moved silently leaves
     nothing.
  2. **The change is recorded, with what it was before.** Without `old_value` a
     history cannot answer "was this always like that", which is the first
     question anyone asks of a surface behaving oddly.
  3. **A clamp is reported, never swallowed.** A caller that believes it set 1.4
     will read the next telemetry as evidence about a bar that was never in
     force — the same class of error as #3739, one layer up.
  4. **An unknown surface is refused before anything is written.** Settings keys
     are free-form strings, so a typo'd surface would write a key nothing reads:
     a change that reports success and alters nothing.
  5. **The tool teaches the procedure that works**, not the one the numbers
     suggest. Asserted on the docstring because the docstring IS the contract an
     agent reads, and the failure it prevents is a model tuning from a
     percentile — which has been measured pointing the wrong way.
"""
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from scribe.services import retrieval_tuning as rt
from tests.helpers import make_mock_session


def _patches(floor=0.72, budget=3):
    """Patch everything `set_dial` touches except the thing under test."""
    session = make_mock_session()
    return session, (
        patch.object(rt, "async_session", MagicMock(return_value=session)),
        patch.object(rt, "set_setting", AsyncMock()),
        patch.object(rt, "floor_for", AsyncMock(return_value=floor)),
        patch.object(rt, "budget_for", AsyncMock(return_value=budget)),
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("reason", ["", "   ", "noisy", "too high"])
async def test_a_change_without_a_real_reason_is_refused(reason):
    """The guardrail. "too high" is a restatement of the change, not a basis."""
    session, ctx = _patches()
    with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError) as e:
        await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)
    # The message has to name the tool that produces a real basis, or the
    # caller's next move is a longer sentence rather than a look at the records.
    assert "near_miss_samples" in str(e.value)
    session.add.assert_not_called()


@pytest.mark.asyncio
async def test_nothing_is_written_when_the_reason_is_refused():
    """Refused BEFORE the setting is touched, not after.

    Writing the value and then raising would leave the number moved and the
    history empty — the exact state this table exists to make impossible.
    """
    session, ctx = _patches()
    with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
         ctx[2], ctx[3]:
        with pytest.raises(ValueError):
            await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason="x")
        setter.assert_not_called()


@pytest.mark.asyncio
async def test_a_good_change_writes_the_setting_and_the_event():
    session, ctx = _patches(floor=0.72)
    reason = ("read prompt_rule's 5 highest declines: 3 were project rules for "
              "another repo, so the bar is doing its job here")
    with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
         ctx[2], ctx[3]:
        out = await rt.set_dial(1, "prompt_rule", "floor", 0.66, reason=reason)

    setter.assert_awaited_once()
    _uid, key, stored = setter.await_args.args
    assert key == "kb_promptrule_threshold" and stored == "0.66"

    event = session.add.call_args.args[0]
    assert event.surface == "prompt_rule" and event.dial == "floor"
    # The before-value is what makes the history answerable.
    assert event.old_value == 0.72 and event.new_value == 0.66
    assert event.reason == reason and event.actor == "model"
    assert out["previous"] == 0.72 and out["applied"] == 0.66
    assert out["clamped"] is False


@pytest.mark.asyncio
@pytest.mark.parametrize("dial, sent, applied", [
    ("floor", 1.4, 1.0),
    ("floor", -0.2, 0.0),
    ("budget", 99, 10),
    ("budget", 0, 1),
])
async def test_a_clamp_is_reported_rather_than_swallowed(dial, sent, applied):
    """Said out loud, because silence here poisons the next reading.

    A caller that believes it set 1.4 treats the following week's telemetry as
    evidence about a bar that never existed, and then moves the dial again to
    fix a problem it invented.
    """
    session, ctx = _patches()
    reason = "checked the refused records for this surface and they were fine"
    with ctx[0], ctx[1], ctx[2], ctx[3]:
        out = await rt.set_dial(1, "auto_inject", dial, sent, reason=reason)
    assert out["applied"] == applied
    assert out["clamped"] is True


@pytest.mark.asyncio
async def test_an_unknown_surface_is_refused_before_anything_is_written():
    session, ctx = _patches()
    with ctx[0], patch.object(rt, "set_setting", AsyncMock()) as setter, \
         ctx[2], ctx[3]:
        with pytest.raises(ValueError):
            await rt.set_dial(1, "pretool_rule", "floor", 0.6,
                              reason="a perfectly good reason that is long enough")
        setter.assert_not_called()
    session.add.assert_not_called()


@pytest.mark.asyncio
@pytest.mark.parametrize("dial", ["threshold", "limit", "", "FLOOR"])
async def test_an_unknown_dial_is_refused(dial):
    """The near-misses are the old vocabulary — `threshold` and `limit` are what
    these were called before this step, so they are exactly what a stale caller
    will send, and a silent no-op there would be indistinguishable from a
    change that did not take."""
    session, ctx = _patches()
    with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
        await rt.set_dial(1, "auto_inject", dial, 0.6,
                          reason="a perfectly good reason that is long enough")
    session.add.assert_not_called()


@pytest.mark.asyncio
async def test_a_human_change_is_distinguishable_from_the_models():
    """Both act as the same user, so the id cannot tell them apart — and "did I
    do this, or did the session?" is the first question the history is asked."""
    session, ctx = _patches()
    with ctx[0], ctx[1], ctx[2], ctx[3]:
        await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="human",
                          reason="operator set this themselves in Settings")
    assert session.add.call_args.args[0].actor == "human"


@pytest.mark.asyncio
async def test_an_unknown_actor_is_refused():
    """Free text here would make the column unreadable within a month."""
    session, ctx = _patches()
    with ctx[0], ctx[1], ctx[2], ctx[3], pytest.raises(ValueError):
        await rt.set_dial(1, "auto_inject", "floor", 0.6, actor="agent",
                          reason="a perfectly good reason that is long enough")


def test_the_tool_teaches_reading_the_records_not_the_percentile():
    """The contract an agent actually reads (rule 167).

    The failure this prevents is a model moving a bar because `near_misses.p90`
    sat close to it. That has been measured pointing the wrong way — 69 declines
    where every percentile said "lower it" and the refused record was a false
    positive — so the docstring has to carry the method, not just the warning.
    """
    from scribe.mcp.tools import retrieval_tuning as tool

    doc = tool.tune_retrieval.__doc__
    assert "near_miss_samples" in doc, "the tool does not name how to get records"
    assert "PERCENTILE ALONE" in doc.upper()
    # And the worked example, because an abstract warning loses to a number.
    assert "69" in doc


def test_every_tool_in_the_module_is_registered():
    from scribe.mcp.tools import retrieval_tuning as tool

    from tests.helpers import FakeMCP

    mcp = FakeMCP()
    tool.register(mcp)
    # Order is the module's, and asserted rather than sorted: an unregistered
    # tool is invisible to every caller, so the list is worth reading literally.
    assert mcp.names == [
        "retrieval_surfaces", "migrate_retrieval_floor",
        "tune_retrieval", "retrieval_tuning_history",
    ]