diff --git a/src/scribe/services/retrieval_tuning.py b/src/scribe/services/retrieval_tuning.py index 98d9da65..12094f3f 100644 --- a/src/scribe/services/retrieval_tuning.py +++ b/src/scribe/services/retrieval_tuning.py @@ -1,32 +1,23 @@ -"""Moving a retrieval surface's floor or budget, with the argument attached (#4102). +"""Moving a retrieval surface's floor or budget, with the argument attached. WHY THIS EXISTS -The operator's decision for milestone 416 step 4: - - "the floor should be chosen and adjusted by the model using it. we've come - back to something either fails or has to be looked at by the user we need a - model consistent surface for the adjustment of these floor values. the user - should be able to touch it but the model should be the thing handling it 9 - times out of 10." - -`retrieval_surfaces` made the six arms describe their numbers the same way. -This module is the write half: one call that moves one dial on one surface, -records what it was, what it became, who moved it and why, and refuses to do -any of that without a reason. +A floor is chosen and adjusted by the model that uses it; the operator can +touch it, but the model handles it most of the time. `retrieval_surfaces` +makes the arms describe their numbers the same way. This module is the write +half: one call that moves one dial on one surface, records what it was, what +it became, who moved it and why, and refuses to do any of that without a +reason. WHY A REASON IS REQUIRED -Because the alternative was tried and measured. The milestone originally listed -self-tuning as a non-goal on the strength of one case: `report_preference` -logged 69 consecutive declines with the refused record 0.0006 under the bar, and -every percentile in the readout said "lower it". Reading the refused record -showed it was rule 77 "Extract intent from loose phrasing" — a false positive — -so lowering the bar would have attached that rule to every completion report -ever written. The statistic and the correct action pointed in opposite -directions. +Because a statistic and the correct action can point in opposite directions. +An arm can log dozens of consecutive declines with the refused record a hair +under the bar, and every percentile in the readout says "lower it" — while +the refused record, once read, turns out to be a false positive that a lower +bar would attach to every matching act. -What separated them was opening the record. A required `reason` is the cheapest +What separates them is opening the record. A required `reason` is the cheapest mechanism that makes that step happen: a caller who must write down why has to have looked, and a caller who writes down something wrong has left the operator a sentence to disagree with. A number moved silently leaves nothing. @@ -62,7 +53,7 @@ logger = logging.getLogger(__name__) DIALS = ("floor", "budget") # The actor for a change nobody made by hand: the shipped default moved -# between releases. Beside "model" and "human" (#4225). +# between releases. Beside "model" and "human". RELEASE_ACTOR = "release" # Long enough to say what was read and what it showed; short enough that nobody @@ -96,17 +87,17 @@ def _calibration(row, s, live: dict) -> dict: Three sources, and they are not interchangeable: - - `tuned` — the dial was moved after #4104 and the event carries its - stamp. The only case where the answer is known. + - `tuned` — the dial was moved and the event carries its stamp. + The only case where the answer is known. - `shipped` — the dial has never been moved, so the number in force is the registry default, and the registry records what THAT was measured against (`Surface.measured_model`). - - `unstamped` — the dial was moved before this step existed. It was + - `unstamped` — the dial was moved before events carried a stamp. It was measured under something; naming it would be inventing a fact, so `stale` is None rather than True or False. "Unknown" and "fine" must not render the same. - `model_changed` and `shape_changed` are reported apart (rule 149) because + `model_changed` and `shape_changed` are reported apart because they call for different responses: a new embedding model means every number is a distance in a geometry that no longer exists, while a re-cut document shape means the same records now embed different text. A caller that only @@ -144,12 +135,12 @@ async def current_settings(user_id: int) -> list[dict]: the reason last given, so the next change argues with the last one instead of overwriting it blind. - From #4104 it also carries `calibration` per dial: the embedding model and + It also carries `calibration` per dial: the embedding model and document shape the number in force was measured in, and whether either has moved since. NOTHING AUTO-RETUNES on the strength of it. A stale stamp says a number is a measurement of a space that no longer exists — it does not - say what the number should be now, and the one time a statistic was allowed - to answer that question it was wrong (see the module docstring). The stamp + say what the number should be now, and a statistic is the wrong thing to + answer that question (see the module docstring). The stamp is here so a reader knows which floors to go and re-measure. """ live = calibration_stamp() @@ -161,10 +152,9 @@ async def current_settings(user_id: int) -> list[dict]: # "The newest two rows" is not "the newest row of each kind": a # surface whose floor was moved three times and whose budget was # moved once returns two floor rows, and the budget change - # disappears. That was a missing reason when this only fed - # `last_change`; since #4104 it is also a WRONG calibration answer — - # a tuned dial reporting as "still on the shipped default", which is - # the one state a reader would not think to check. + # disappears — a missing reason in `last_change` and a WRONG + # calibration answer: a tuned dial reporting as "still on the + # shipped default", the one state a reader would not think to check. last = {} for dial in DIALS: row = ( @@ -172,17 +162,16 @@ async def current_settings(user_id: int) -> list[dict]: select(RetrievalTuningEvent) .where( RetrievalTuningEvent.surface == name, - # THIS USER'S CHANGES *OR* A RELEASE'S (#4225). + # THIS USER'S CHANGES *OR* A RELEASE'S. # A release's rows carry no user id because no user # made them, and taking the newest of the two is # what makes the answer true: a dial the operator # has tuned is explained by their change, and an # untouched one by the release that last shipped - # its default. Filtering to the user alone reported - # "still on the shipped starting point" for a - # default that had in fact moved — the one state a - # reader would not think to check, which is the - # same sentence #4104 wrote about the bug above. + # its default. Filtering to the user alone would + # report "still on the shipped starting point" for + # a default that had in fact moved — the one state + # a reader would not think to check. or_( RetrievalTuningEvent.user_id == user_id, RetrievalTuningEvent.user_id.is_(None), @@ -270,7 +259,7 @@ async def set_dial( await set_setting(user_id, key, stored) async with async_session() as session: - # Stamped with the space this number was chosen in (#4104). Read at + # Stamped with the space this number was chosen in. Read at # write time rather than passed in: the caller measuring a floor and # the caller recording it are the same call, so there is no window in # which they could disagree. @@ -301,7 +290,7 @@ async def set_dial( async def record_release_defaults() -> list[dict]: - """Write down that a RELEASE moved a shipped default (#4225). + """Write down that a RELEASE moved a shipped default. THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a person or a model choosing a number. It is silent about the other way a @@ -310,12 +299,10 @@ async def record_release_defaults() -> list[dict]: one of those consumers is `band_hugs_floor`, which compares a percentile against a floor and has no way to notice they come from different regimes. - Measured on the instance that found this: `write_path_rule` went 0.68 -> - 0.72 on 2026-09-02 as a shipped default, and a 30-day window opening - 2026-08-22 therefore held six days of calls made under the old bar. The - warning reported the band sitting "-0.0216 above" its floor — a negative - distance above something, which is what a two-population comparison looks - like when it finally says so out loud. + A window that straddles a shipped change holds calls made under two bars, + and the warning can report a band sitting "-0.0216 above" its floor — a + negative distance above something, which is what a two-population + comparison looks like when it finally says so out loud. `user_id IS NULL`, because no user did this. A release acts on every account that has not overridden the dial, and writing one row per user @@ -423,18 +410,17 @@ async def floor_moves_since(since) -> dict[str, str]: async def floor_history_gaps(since) -> dict[str, str | None]: """Surfaces whose floor history does NOT cover a window starting at `since`. - The companion `floor_moves_since` needs, and the hole #4225's fix left. - That function answers "did this arm's floor move inside the window", and an - empty answer was being read as "no, it held steady". Those are the same - answer only while the LEDGER covers the window. Before the ledger existed - there are no rows for any arm, so every arm reported "no move" for a period - nobody had recorded — absence of evidence arriving as evidence of absence. + The companion `floor_moves_since` needs. That function answers "did this + arm's floor move inside the window", and an empty answer must not be read + as "no, it held steady". Those are the same answer only while the LEDGER + covers the window. Before an arm's first ledger row there are no rows, so + it would report "no move" for a period nobody recorded — absence of + evidence arriving as evidence of absence. - It is not hypothetical, and it is not a one-off: every install passes - through it. The ledger's first row for an arm is written when that install - first boots the release that records baselines, so for a window longer + Every install passes through this. The ledger's first row for an arm is + written when that install first records baselines, so for a window longer than the install is old — or than the arm is old, for an arm added - later — there is no history to consult, and the band check was reading + later — there is no history to consult, and the band check must not read that silence as a clean bill of health. Returns ONLY the arms with a gap, mapped to the ISO timestamp from which @@ -488,7 +474,7 @@ async def tuning_history( RetrievalTuningEvent.user_id == user_id, # Shipped-default changes, which belong to no account and # would otherwise be missing from the one surface built - # for asking why a number is where it is (#4225). + # for asking why a number is where it is. RetrievalTuningEvent.user_id.is_(None), )) .order_by(RetrievalTuningEvent.created_at.desc())