FabledGrimoire/src/scribe/services/retrieval_tuning.py

"""Moving a retrieval surface's floor or budget, with the argument attached (#4102).

WHY THIS EXISTS

The operator's decision for milestone 416 step 4:

    "the floor should be chosen and adjusted by the model using it. we've come
    back to something either fails or has to be looked at by the user we need a
    model consistent surface for the adjustment of these floor values. the user
    should be able to touch it but the model should be the thing handling it 9
    times out of 10."

`retrieval_surfaces` made the six arms describe their numbers the same way.
This module is the write half: one call that moves one dial on one surface,
records what it was, what it became, who moved it and why, and refuses to do
any of that without a reason.

WHY A REASON IS REQUIRED

Because the alternative was tried and measured. The milestone originally listed
self-tuning as a non-goal on the strength of one case: `report_preference`
logged 69 consecutive declines with the refused record 0.0006 under the bar, and
every percentile in the readout said "lower it". Reading the refused record
showed it was rule 77 "Extract intent from loose phrasing" — a false positive —
so lowering the bar would have attached that rule to every completion report
ever written. The statistic and the correct action pointed in opposite
directions.

What separated them was opening the record. A required `reason` is the cheapest
mechanism that makes that step happen: a caller who must write down why has to
have looked, and a caller who writes down something wrong has left the operator
a sentence to disagree with. A number moved silently leaves nothing.

WHAT THIS DELIBERATELY DOES NOT DO

It does not decide anything itself. There is no rule here that reads a
percentile and picks a value, and that absence is the design — the non-goal that
survived is *statistical* auto-tuning, precisely because the statistic was the
thing that was wrong. The judgement stays with the reader; this module only
makes the judgement recordable and reversible.
"""
from __future__ import annotations

import logging

from sqlalchemy import select

from scribe.models import async_session
from scribe.models.retrieval_tuning import RetrievalTuningEvent
from scribe.services.embeddings import calibration_stamp
from scribe.services.retrieval_surfaces import (
    MAX_BUDGET,
    budget_for,
    floor_for,
    get_surface,
    surface_names,
)
from scribe.services.settings import set_setting

logger = logging.getLogger(__name__)

DIALS = ("floor", "budget")

# Long enough to say what was read and what it showed; short enough that nobody
# pastes a telemetry dump in. The number is not a measurement — it is the point
# at which "0.66" stops being an acceptable answer to "why".
_MIN_REASON_CHARS = 20


def _clean_reason(reason: str) -> str:
    """The guardrail, enforced here rather than by the column.

    `reason` is NOT NULL in the schema, which "" satisfies. A required field
    that accepts an empty string is a formality, and this one is the whole
    mechanism — see the module docstring.
    """
    text = (reason or "").strip()
    if len(text) < _MIN_REASON_CHARS:
        raise ValueError(
            "reason is required, and has to say what you read. A floor moved "
            "without a stated basis is a number nobody can review or revert. "
            "Name the evidence: which surface's telemetry, and what the refused "
            "records actually were — `retrieval_telemetry(near_miss_samples=N)` "
            "returns them by id, and reading them is the step that separates a "
            "real miss from a bar doing its job."
        )
    return text


def _calibration(row, s, live: dict) -> dict:
    """What space one dial's number was chosen in, and whether that space moved.

    Three sources, and they are not interchangeable:

    - `tuned`     — the dial was moved after #4104 and the event carries its
                    stamp. The only case where the answer is known.
    - `shipped`   — the dial has never been moved, so the number in force is
                    the registry default, and the registry records what THAT
                    was measured against (`Surface.measured_model`).
    - `unstamped` — the dial was moved before this step existed. It was
                    measured under something; naming it would be inventing a
                    fact, so `stale` is None rather than True or False.
                    "Unknown" and "fine" must not render the same.

    `model_changed` and `shape_changed` are reported apart (rule 149) because
    they call for different responses: a new embedding model means every number
    is a distance in a geometry that no longer exists, while a re-cut document
    shape means the same records now embed different text. A caller that only
    ever sees `stale: true` cannot tell those apart.
    """
    if row is None:
        model, shape, source = s.measured_model, s.measured_shape, "shipped"
    elif row.embedding_model is None and row.shape_version is None:
        return {
            "source": "unstamped", "embedding_model": None,
            "shape_version": None, "model_changed": None,
            "shape_changed": None, "stale": None,
        }
    else:
        model, shape, source = row.embedding_model, row.shape_version, "tuned"

    model_changed = model != live["embedding_model"]
    shape_changed = shape != live["shape_version"]
    return {
        "source": source,
        "embedding_model": model,
        "shape_version": shape,
        "model_changed": model_changed,
        "shape_changed": shape_changed,
        "stale": model_changed or shape_changed,
    }


async def current_settings(user_id: int) -> list[dict]:
    """Every tunable surface with its live pair and its last stated reason.

    The read side of the tuning surface, and shaped for a reader who is about to
    change something: the value, what the arm asks and over what corpus and how
    often — because a floor cannot be moved sensibly without those three — and
    the reason last given, so the next change argues with the last one instead
    of overwriting it blind.

    From #4104 it also carries `calibration` per dial: the embedding model and
    document shape the number in force was measured in, and whether either has
    moved since. NOTHING AUTO-RETUNES on the strength of it. A stale stamp says
    a number is a measurement of a space that no longer exists — it does not
    say what the number should be now, and the one time a statistic was allowed
    to answer that question it was wrong (see the module docstring). The stamp
    is here so a reader knows which floors to go and re-measure.
    """
    live = calibration_stamp()
    out: list[dict] = []
    async with async_session() as session:
        for name in surface_names():
            s = get_surface(name)
            # ONE QUERY PER DIAL, not one `limit(len(DIALS))` over both.
            # "The newest two rows" is not "the newest row of each kind": a
            # surface whose floor was moved three times and whose budget was
            # moved once returns two floor rows, and the budget change
            # disappears. That was a missing reason when this only fed
            # `last_change`; since #4104 it is also a WRONG calibration answer —
            # a tuned dial reporting as "still on the shipped default", which is
            # the one state a reader would not think to check.
            last = {}
            for dial in DIALS:
                row = (
                    await session.execute(
                        select(RetrievalTuningEvent)
                        .where(
                            RetrievalTuningEvent.surface == name,
                            RetrievalTuningEvent.user_id == user_id,
                            RetrievalTuningEvent.dial == dial,
                        )
                        .order_by(RetrievalTuningEvent.created_at.desc())
                        .limit(1)
                    )
                ).scalars().first()
                if row is not None:
                    last[dial] = row
            out.append({
                "surface": name,
                "floor": await floor_for(user_id, name),
                "budget": await budget_for(user_id, name),
                "floor_default": s.floor_default,
                "budget_default": s.budget_default,
                "asks": s.asks,
                "over": s.over,
                "fires": s.fires,
                # Absent rather than empty when a surface has never been moved:
                # "still on the shipped starting point" is a real state and
                # should not render as a blank reason somebody wrote.
                "last_change": {
                    dial: last[dial].to_dict() for dial in DIALS if dial in last
                },
                # Always present for BOTH dials, unlike `last_change`: an
                # untouched dial still has a number in force, and that number
                # was still measured in some space. Reported, never acted on —
                # see the note below on why nothing auto-retunes.
                "calibration": {
                    dial: _calibration(last.get(dial), s, live) for dial in DIALS
                },
            })
    return out


async def set_dial(
    user_id: int,
    surface: str,
    dial: str,
    value: float,
    *,
    reason: str,
    actor: str = "model",
) -> dict:
    """Move one dial on one surface, recording the change and its argument.

    Returns the applied value alongside the previous one, so a caller can see
    that a clamp bit rather than assuming the number it sent is the number in
    force — the failure being avoided is a tool reporting success for a value
    the registry silently corrected.
    """
    s = get_surface(surface)                      # refuses an unknown name
    if dial not in DIALS:
        raise ValueError(f"dial must be one of {DIALS}, got {dial!r}")
    if actor not in ("model", "human"):
        raise ValueError(f"actor must be 'model' or 'human', got {actor!r}")
    text = _clean_reason(reason)

    if dial == "floor":
        old = await floor_for(user_id, surface)
        applied = min(1.0, max(0.0, float(value)))
        key, stored = s.floor_key, str(applied)
    else:
        old = float(await budget_for(user_id, surface))
        applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
        key, stored = s.budget_key, str(int(applied))

    # A change to the value it already has is not a change, and must not be
    # written. The Settings form re-sends every field on every save, so without
    # this the history fills with rows saying the operator set six dials to the
    # numbers they were already on — and a history nobody can skim is one
    # nobody reads, which costs the surface its entire purpose.
    #
    # Reported rather than silently skipped, so a caller that expected to move
    # something learns that it did not.
    if abs(applied - old) < 1e-9:
        return {
            "surface": surface, "dial": dial, "previous": old,
            "applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
            "reason": text, "actor": actor, "unchanged": True,
        }

    await set_setting(user_id, key, stored)
    async with async_session() as session:
        # Stamped with the space this number was chosen in (#4104). Read at
        # write time rather than passed in: the caller measuring a floor and
        # the caller recording it are the same call, so there is no window in
        # which they could disagree.
        stamp = calibration_stamp()
        session.add(RetrievalTuningEvent(
            user_id=user_id, surface=surface, dial=dial,
            old_value=old, new_value=applied, actor=actor, reason=text,
            embedding_model=stamp["embedding_model"],
            shape_version=stamp["shape_version"],
        ))
        await session.commit()

    return {
        "surface": surface,
        "dial": dial,
        "previous": old,
        "applied": applied,
        # True when the registry corrected what was asked for. Said out loud
        # because a caller that believes it set 1.4 will read the next
        # telemetry as evidence about a bar that was never in force.
        "clamped": abs(applied - float(value)) > 1e-9,
        "reason": text,
        "actor": actor,
        # Always present, both ways round: a caller that has to test for the
        # key's absence to learn the answer will eventually forget to.
        "unchanged": False,
    }


async def tuning_history(
    user_id: int, *, surface: str | None = None, limit: int = 20
) -> list[dict]:
    """What has been moved, newest first — the operator's review surface.

    Scoped to one surface when asked, because the question is almost always
    "why is THIS arm set like this", and an unscoped list buries one surface's
    two changes under another's twenty.
    """
    if surface is not None:
        get_surface(surface)                      # refuse a typo on the read too
    async with async_session() as session:
        q = (
            select(RetrievalTuningEvent)
            .where(RetrievalTuningEvent.user_id == user_id)
            .order_by(RetrievalTuningEvent.created_at.desc())
            .limit(max(1, min(int(limit), 200)))
        )
        if surface is not None:
            q = q.where(RetrievalTuningEvent.surface == surface)
        rows = (await session.execute(q)).scalars().all()
    return [r.to_dict() for r in rows]