bvandeusenandClaude Opus 5.5
6299335254
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 14s
CI & Build / TypeScript typecheck (push) Successful in 53s
CI & Build / integration (push) Successful in 1m9s
CI & Build / Python tests (push) Successful in 1m58s
CI & Build / Build & push image (push) Successful in 33s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
499 lines
22 KiB
Python
499 lines
22 KiB
Python
"""Moving a retrieval surface's floor or budget, with the argument attached.
|
|
|
|
WHY THIS EXISTS
|
|
|
|
A floor is chosen and adjusted by the model that uses it; the operator can
|
|
touch it, but the model handles it most of the time. `retrieval_surfaces`
|
|
makes the arms describe their numbers the same way. This module is the write
|
|
half: one call that moves one dial on one surface, records what it was, what
|
|
it became, who moved it and why, and refuses to do any of that without a
|
|
reason.
|
|
|
|
WHY A REASON IS REQUIRED
|
|
|
|
Because a statistic and the correct action can point in opposite directions.
|
|
An arm can log dozens of consecutive declines with the refused record a hair
|
|
under the bar, and every percentile in the readout says "lower it" — while
|
|
the refused record, once read, turns out to be a false positive that a lower
|
|
bar would attach to every matching act.
|
|
|
|
What separates them is opening the record. A required `reason` is the cheapest
|
|
mechanism that makes that step happen: a caller who must write down why has to
|
|
have looked, and a caller who writes down something wrong has left the operator
|
|
a sentence to disagree with. A number moved silently leaves nothing.
|
|
|
|
WHAT THIS DELIBERATELY DOES NOT DO
|
|
|
|
It does not decide anything itself. There is no rule here that reads a
|
|
percentile and picks a value, and that absence is the design — the non-goal that
|
|
survived is *statistical* auto-tuning, precisely because the statistic was the
|
|
thing that was wrong. The judgement stays with the reader; this module only
|
|
makes the judgement recordable and reversible.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from sqlalchemy import func, or_, select
|
|
|
|
from grimoire.models import async_session
|
|
from grimoire.models.retrieval_tuning import RetrievalTuningEvent
|
|
from grimoire.services.embeddings import calibration_stamp
|
|
from grimoire.services.retrieval_surfaces import (
|
|
MAX_BUDGET,
|
|
budget_for,
|
|
floor_for,
|
|
get_surface,
|
|
sharing_floor,
|
|
surface_names,
|
|
)
|
|
from grimoire.services.settings import set_setting
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DIALS = ("floor", "budget")
|
|
|
|
# The actor for a change nobody made by hand: the shipped default moved
|
|
# between releases. Beside "model" and "human".
|
|
RELEASE_ACTOR = "release"
|
|
|
|
# Long enough to say what was read and what it showed; short enough that nobody
|
|
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
|
# at which "0.66" stops being an acceptable answer to "why".
|
|
_MIN_REASON_CHARS = 20
|
|
|
|
|
|
def _clean_reason(reason: str) -> str:
|
|
"""The guardrail, enforced here rather than by the column.
|
|
|
|
`reason` is NOT NULL in the schema, which "" satisfies. A required field
|
|
that accepts an empty string is a formality, and this one is the whole
|
|
mechanism — see the module docstring.
|
|
"""
|
|
text = (reason or "").strip()
|
|
if len(text) < _MIN_REASON_CHARS:
|
|
raise ValueError(
|
|
"reason is required, and has to say what you read. A floor moved "
|
|
"without a stated basis is a number nobody can review or revert. "
|
|
"Name the evidence: which surface's telemetry, and what the refused "
|
|
"records actually were — `retrieval_telemetry(near_miss_samples=N)` "
|
|
"returns them by id, and reading them is the step that separates a "
|
|
"real miss from a bar doing its job."
|
|
)
|
|
return text
|
|
|
|
|
|
def _calibration(row, s, live: dict) -> dict:
|
|
"""What space one dial's number was chosen in, and whether that space moved.
|
|
|
|
Three sources, and they are not interchangeable:
|
|
|
|
- `tuned` — the dial was moved and the event carries its stamp.
|
|
The only case where the answer is known.
|
|
- `shipped` — the dial has never been moved, so the number in force is
|
|
the registry default, and the registry records what THAT
|
|
was measured against (`Surface.measured_model`).
|
|
- `unstamped` — the dial was moved before events carried a stamp. It was
|
|
measured under something; naming it would be inventing a
|
|
fact, so `stale` is None rather than True or False.
|
|
"Unknown" and "fine" must not render the same.
|
|
|
|
`model_changed` and `shape_changed` are reported apart because
|
|
they call for different responses: a new embedding model means every number
|
|
is a distance in a geometry that no longer exists, while a re-cut document
|
|
shape means the same records now embed different text. A caller that only
|
|
ever sees `stale: true` cannot tell those apart.
|
|
"""
|
|
if row is None:
|
|
model, shape, source = s.measured_model, s.measured_shape, "shipped"
|
|
elif row.embedding_model is None and row.shape_version is None:
|
|
return {
|
|
"source": "unstamped", "embedding_model": None,
|
|
"shape_version": None, "model_changed": None,
|
|
"shape_changed": None, "stale": None,
|
|
}
|
|
else:
|
|
model, shape, source = row.embedding_model, row.shape_version, "tuned"
|
|
|
|
model_changed = model != live["embedding_model"]
|
|
shape_changed = shape != live["shape_version"]
|
|
return {
|
|
"source": source,
|
|
"embedding_model": model,
|
|
"shape_version": shape,
|
|
"model_changed": model_changed,
|
|
"shape_changed": shape_changed,
|
|
"stale": model_changed or shape_changed,
|
|
}
|
|
|
|
|
|
async def current_settings(user_id: int) -> list[dict]:
|
|
"""Every tunable surface with its live pair and its last stated reason.
|
|
|
|
The read side of the tuning surface, and shaped for a reader who is about to
|
|
change something: the value, what the arm asks and over what corpus and how
|
|
often — because a floor cannot be moved sensibly without those three — and
|
|
the reason last given, so the next change argues with the last one instead
|
|
of overwriting it blind.
|
|
|
|
It also carries `calibration` per dial: the embedding model and
|
|
document shape the number in force was measured in, and whether either has
|
|
moved since. NOTHING AUTO-RETUNES on the strength of it. A stale stamp says
|
|
a number is a measurement of a space that no longer exists — it does not
|
|
say what the number should be now, and a statistic is the wrong thing to
|
|
answer that question (see the module docstring). The stamp
|
|
is here so a reader knows which floors to go and re-measure.
|
|
"""
|
|
live = calibration_stamp()
|
|
out: list[dict] = []
|
|
async with async_session() as session:
|
|
for name in surface_names():
|
|
s = get_surface(name)
|
|
# ONE QUERY PER DIAL, not one `limit(len(DIALS))` over both.
|
|
# "The newest two rows" is not "the newest row of each kind": a
|
|
# surface whose floor was moved three times and whose budget was
|
|
# moved once returns two floor rows, and the budget change
|
|
# disappears — a missing reason in `last_change` and a WRONG
|
|
# calibration answer: a tuned dial reporting as "still on the
|
|
# shipped default", the one state a reader would not think to check.
|
|
last = {}
|
|
for dial in DIALS:
|
|
row = (
|
|
await session.execute(
|
|
select(RetrievalTuningEvent)
|
|
.where(
|
|
RetrievalTuningEvent.surface == name,
|
|
# THIS USER'S CHANGES *OR* A RELEASE'S.
|
|
# A release's rows carry no user id because no user
|
|
# made them, and taking the newest of the two is
|
|
# what makes the answer true: a dial the operator
|
|
# has tuned is explained by their change, and an
|
|
# untouched one by the release that last shipped
|
|
# its default. Filtering to the user alone would
|
|
# report "still on the shipped starting point" for
|
|
# a default that had in fact moved — the one state
|
|
# a reader would not think to check.
|
|
or_(
|
|
RetrievalTuningEvent.user_id == user_id,
|
|
RetrievalTuningEvent.user_id.is_(None),
|
|
),
|
|
RetrievalTuningEvent.dial == dial,
|
|
)
|
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
|
.limit(1)
|
|
)
|
|
).scalars().first()
|
|
if row is not None:
|
|
last[dial] = row
|
|
out.append({
|
|
"surface": name,
|
|
"floor": await floor_for(user_id, name),
|
|
"budget": await budget_for(user_id, name),
|
|
"floor_default": s.floor_default,
|
|
"budget_default": s.budget_default,
|
|
"asks": s.asks,
|
|
"over": s.over,
|
|
"fires": s.fires,
|
|
# Absent rather than empty when a surface has never been moved:
|
|
# "still on the shipped starting point" is a real state and
|
|
# should not render as a blank reason somebody wrote.
|
|
"last_change": {
|
|
dial: last[dial].to_dict() for dial in DIALS if dial in last
|
|
},
|
|
# Always present for BOTH dials, unlike `last_change`: an
|
|
# untouched dial still has a number in force, and that number
|
|
# was still measured in some space. Reported, never acted on —
|
|
# see the note below on why nothing auto-retunes.
|
|
"calibration": {
|
|
dial: _calibration(last.get(dial), s, live) for dial in DIALS
|
|
},
|
|
})
|
|
return out
|
|
|
|
|
|
async def set_dial(
|
|
user_id: int,
|
|
surface: str,
|
|
dial: str,
|
|
value: float,
|
|
*,
|
|
reason: str,
|
|
actor: str = "model",
|
|
) -> dict:
|
|
"""Move one dial on one surface, recording the change and its argument.
|
|
|
|
Returns the applied value alongside the previous one, so a caller can see
|
|
that a clamp bit rather than assuming the number it sent is the number in
|
|
force — the failure being avoided is a tool reporting success for a value
|
|
the registry silently corrected.
|
|
"""
|
|
s = get_surface(surface) # refuses an unknown name
|
|
if dial not in DIALS:
|
|
raise ValueError(f"dial must be one of {DIALS}, got {dial!r}")
|
|
if actor not in ("model", "human"):
|
|
raise ValueError(f"actor must be 'model' or 'human', got {actor!r}")
|
|
text = _clean_reason(reason)
|
|
|
|
if dial == "floor":
|
|
old = await floor_for(user_id, surface)
|
|
applied = min(1.0, max(0.0, float(value)))
|
|
key, stored = s.floor_key, str(applied)
|
|
else:
|
|
old = float(await budget_for(user_id, surface))
|
|
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
|
|
key, stored = s.budget_key, str(int(applied))
|
|
|
|
# A change to the value it already has is not a change, and must not be
|
|
# written. The Settings form re-sends every field on every save, so without
|
|
# this the history fills with rows saying the operator set six dials to the
|
|
# numbers they were already on — and a history nobody can skim is one
|
|
# nobody reads, which costs the surface its entire purpose.
|
|
#
|
|
# Reported rather than silently skipped, so a caller that expected to move
|
|
# something learns that it did not.
|
|
if abs(applied - old) < 1e-9:
|
|
return {
|
|
"surface": surface, "dial": dial, "previous": old,
|
|
"applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
|
|
"reason": text, "actor": actor, "unchanged": True,
|
|
}
|
|
|
|
# A shared floor moves every surface that reads it, and each one's history
|
|
# has to show the move, or its telemetry changes with no event to explain it.
|
|
# The surface the caller named is recorded first, its siblings after it.
|
|
moved = [surface] + (
|
|
[n for n in sharing_floor(surface) if n != surface] if dial == "floor" else []
|
|
)
|
|
|
|
await set_setting(user_id, key, stored)
|
|
async with async_session() as session:
|
|
# Stamped with the space this number was chosen in. Read at
|
|
# write time rather than passed in: the caller measuring a floor and
|
|
# the caller recording it are the same call, so there is no window in
|
|
# which they could disagree.
|
|
stamp = calibration_stamp()
|
|
for name in moved:
|
|
session.add(RetrievalTuningEvent(
|
|
user_id=user_id, surface=name, dial=dial,
|
|
old_value=old, new_value=applied, actor=actor, reason=text,
|
|
embedding_model=stamp["embedding_model"],
|
|
shape_version=stamp["shape_version"],
|
|
))
|
|
await session.commit()
|
|
|
|
return {
|
|
"surface": surface,
|
|
"dial": dial,
|
|
"previous": old,
|
|
"applied": applied,
|
|
# True when the registry corrected what was asked for. Said out loud
|
|
# because a caller that believes it set 1.4 will read the next
|
|
# telemetry as evidence about a bar that was never in force.
|
|
"clamped": abs(applied - float(value)) > 1e-9,
|
|
"reason": text,
|
|
"actor": actor,
|
|
# Always present, both ways round: a caller that has to test for the
|
|
# key's absence to learn the answer will eventually forget to.
|
|
"unchanged": False,
|
|
# The other surfaces this move carried with it: a shared floor is one
|
|
# number, and a caller tuning one rule arm has tuned them all.
|
|
"moved_with": [n for n in moved if n != surface],
|
|
}
|
|
|
|
|
|
async def record_release_defaults() -> list[dict]:
|
|
"""Write down that a RELEASE moved a shipped default.
|
|
|
|
THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a
|
|
person or a model choosing a number. It is silent about the other way a
|
|
floor moves, which is somebody editing `floor_default` in the registry and
|
|
shipping it. That change is invisible to every consumer of this table, and
|
|
one of those consumers is `band_hugs_floor`, which compares a percentile
|
|
against a floor and has no way to notice they come from different regimes.
|
|
|
|
A window that straddles a shipped change holds calls made under two bars,
|
|
and the warning can report a band sitting "-0.0216 above" its floor — a
|
|
negative distance above something, which is what a two-population
|
|
comparison looks like when it finally says so out loud.
|
|
|
|
`user_id IS NULL`, because no user did this. A release acts on every
|
|
account that has not overridden the dial, and writing one row per user
|
|
would both multiply the row and misattribute it. Readers take the newest of
|
|
(this user's own change, the release's) — a dial the operator has tuned is
|
|
explained by their change, and an untouched one by the release.
|
|
|
|
`actor="release"`, a third value beside "model" and "human". The column is
|
|
Text with no CHECK precisely so a new kind of actor is not a migration —
|
|
the model's own comment says so, and this is the case it anticipated.
|
|
|
|
THE FIRST SIGHTING IS A BASELINE, NOT A CHANGE, and is written with
|
|
`old_value=None`. Nothing moved; the row exists so that the NEXT release
|
|
has something to be different from. That distinction is load-bearing
|
|
downstream: a warning suspends itself on a genuine move and must not
|
|
suspend itself on a fresh install's baseline, and it tells the two apart by
|
|
exactly this null.
|
|
|
|
Idempotent, and safe to call on every boot: a default that matches the last
|
|
recorded one writes nothing.
|
|
"""
|
|
written: list[dict] = []
|
|
stamp = calibration_stamp()
|
|
async with async_session() as session:
|
|
for name in surface_names():
|
|
s = get_surface(name)
|
|
for dial, shipped in (
|
|
("floor", float(s.floor_default)),
|
|
("budget", float(s.budget_default)),
|
|
):
|
|
last = (
|
|
await session.execute(
|
|
select(RetrievalTuningEvent)
|
|
.where(
|
|
RetrievalTuningEvent.surface == name,
|
|
RetrievalTuningEvent.user_id.is_(None),
|
|
RetrievalTuningEvent.actor == RELEASE_ACTOR,
|
|
RetrievalTuningEvent.dial == dial,
|
|
)
|
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
|
.limit(1)
|
|
)
|
|
).scalars().first()
|
|
if last is not None and abs(float(last.new_value) - shipped) < 1e-9:
|
|
continue
|
|
old = None if last is None else float(last.new_value)
|
|
reason = (
|
|
f"Baseline: this release ships {name}'s {dial} at {shipped}. "
|
|
f"Recorded so a later change to the shipped value has a "
|
|
f"date and a predecessor to be measured against; nothing "
|
|
f"moved here."
|
|
if old is None else
|
|
f"A release moved {name}'s shipped {dial} from {old} to "
|
|
f"{shipped}. Not a dial turn — the default in the registry "
|
|
f"changed, so calls logged either side of this date were "
|
|
f"made under different bars and cannot be pooled."
|
|
)
|
|
session.add(RetrievalTuningEvent(
|
|
user_id=None, surface=name, dial=dial,
|
|
old_value=old, new_value=shipped,
|
|
actor=RELEASE_ACTOR, reason=reason,
|
|
embedding_model=stamp["embedding_model"],
|
|
shape_version=stamp["shape_version"],
|
|
))
|
|
written.append({
|
|
"surface": name, "dial": dial,
|
|
"old_value": old, "new_value": shipped,
|
|
"baseline": old is None,
|
|
})
|
|
if written:
|
|
await session.commit()
|
|
return written
|
|
|
|
|
|
async def floor_moves_since(since) -> dict[str, str]:
|
|
"""Surfaces whose floor genuinely MOVED at or after `since`.
|
|
|
|
Genuinely: `old_value IS NOT NULL`, so a baseline row — which records a
|
|
default without changing it — does not read as a change. Returns surface ->
|
|
ISO timestamp of the newest such move, for a reader deciding whether a
|
|
window's numbers can be pooled.
|
|
|
|
Covers both kinds of move, a release's and this user's, because the
|
|
question is about the SAMPLE rather than about who is responsible for it:
|
|
a floor that moved mid-window splits the calls either way round.
|
|
"""
|
|
out: dict[str, str] = {}
|
|
async with async_session() as session:
|
|
rows = (
|
|
await session.execute(
|
|
select(RetrievalTuningEvent)
|
|
.where(
|
|
RetrievalTuningEvent.dial == "floor",
|
|
RetrievalTuningEvent.old_value.isnot(None),
|
|
RetrievalTuningEvent.created_at >= since,
|
|
)
|
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
|
)
|
|
).scalars().all()
|
|
for row in rows:
|
|
out.setdefault(row.surface, row.created_at.isoformat())
|
|
return out
|
|
|
|
|
|
async def floor_history_gaps(since) -> dict[str, str | None]:
|
|
"""Surfaces whose floor history does NOT cover a window starting at `since`.
|
|
|
|
The companion `floor_moves_since` needs. That function answers "did this
|
|
arm's floor move inside the window", and an empty answer must not be read
|
|
as "no, it held steady". Those are the same answer only while the LEDGER
|
|
covers the window. Before an arm's first ledger row there are no rows, so
|
|
it would report "no move" for a period nobody recorded — absence of
|
|
evidence arriving as evidence of absence.
|
|
|
|
Every install passes through this. The ledger's first row for an arm is
|
|
written when that install first records baselines, so for a window longer
|
|
than the install is old — or than the arm is old, for an arm added
|
|
later — there is no history to consult, and the band check must not read
|
|
that silence as a clean bill of health.
|
|
|
|
Returns ONLY the arms with a gap, mapped to the ISO timestamp from which
|
|
their floor IS knowable — or None when the arm has no floor history at all.
|
|
An arm absent from the result is fully covered and can be judged. Shaped
|
|
that way so a caller loops over what it is already judging and asks one
|
|
question per arm, rather than reasoning about the ledger itself.
|
|
|
|
Every row counts here, baselines included, which is the one place
|
|
`floor_moves_since` and this deliberately disagree. A baseline is not a
|
|
change — so it is not a move — but it IS the ledger observing the arm, and
|
|
from that moment on silence genuinely means the floor held.
|
|
"""
|
|
out: dict[str, str | None] = {}
|
|
async with async_session() as session:
|
|
rows = (
|
|
await session.execute(
|
|
select(
|
|
RetrievalTuningEvent.surface,
|
|
func.min(RetrievalTuningEvent.created_at),
|
|
)
|
|
.where(RetrievalTuningEvent.dial == "floor")
|
|
.group_by(RetrievalTuningEvent.surface)
|
|
)
|
|
).all()
|
|
earliest = {surface: first for surface, first in rows}
|
|
for name in surface_names():
|
|
first = earliest.get(name)
|
|
if first is None:
|
|
out[name] = None
|
|
elif first > since:
|
|
out[name] = first.isoformat()
|
|
return out
|
|
|
|
|
|
async def tuning_history(
|
|
user_id: int, *, surface: str | None = None, limit: int = 20
|
|
) -> list[dict]:
|
|
"""What has been moved, newest first — the operator's review surface.
|
|
|
|
Scoped to one surface when asked, because the question is almost always
|
|
"why is THIS arm set like this", and an unscoped list buries one surface's
|
|
two changes under another's twenty.
|
|
"""
|
|
if surface is not None:
|
|
get_surface(surface) # refuse a typo on the read too
|
|
async with async_session() as session:
|
|
q = (
|
|
select(RetrievalTuningEvent)
|
|
.where(or_(
|
|
RetrievalTuningEvent.user_id == user_id,
|
|
# Shipped-default changes, which belong to no account and
|
|
# would otherwise be missing from the one surface built
|
|
# for asking why a number is where it is.
|
|
RetrievalTuningEvent.user_id.is_(None),
|
|
))
|
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
|
.limit(max(1, min(int(limit), 200)))
|
|
)
|
|
if surface is not None:
|
|
q = q.where(RetrievalTuningEvent.surface == surface)
|
|
rows = (await session.execute(q)).scalars().all()
|
|
return [r.to_dict() for r in rows]
|