Rule outcomes, the contract hint, and four extractor/backup fixes #174
@@ -253,6 +253,21 @@ async function loadTuningHistory() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Who moved a dial. Three answers, not two (#4225).
|
||||||
|
*
|
||||||
|
* `release` means the shipped default itself changed between versions — no
|
||||||
|
* user turned anything. Folding that into the `human ? 'you' : 'Claude'`
|
||||||
|
* fallback labelled it "Claude", which tells the operator the session moved a
|
||||||
|
* floor it never touched. Misattribution is the one failure the `actor`
|
||||||
|
* column exists to prevent, so an unknown value says so rather than guessing.
|
||||||
|
*/
|
||||||
|
function actorLabel(actor: string): string {
|
||||||
|
if (actor === "human") return "you";
|
||||||
|
if (actor === "release") return "this release";
|
||||||
|
if (actor === "model") return "Claude";
|
||||||
|
return actor;
|
||||||
|
}
|
||||||
|
|
||||||
async function saveKbInject() {
|
async function saveKbInject() {
|
||||||
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
||||||
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
||||||
@@ -1925,8 +1940,11 @@ async function deleteUser(userId: number) {
|
|||||||
<template v-else>set to</template>
|
<template v-else>set to</template>
|
||||||
{{ ev.new_value }}
|
{{ ev.new_value }}
|
||||||
</span>
|
</span>
|
||||||
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }">
|
<span
|
||||||
{{ ev.actor === 'human' ? 'you' : 'Claude' }}
|
class="tuning-actor"
|
||||||
|
:class="{ 'is-human': ev.actor === 'human', 'is-release': ev.actor === 'release' }"
|
||||||
|
>
|
||||||
|
{{ actorLabel(ev.actor) }}
|
||||||
</span>
|
</span>
|
||||||
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
|
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -4236,6 +4254,14 @@ async function deleteUser(userId: number) {
|
|||||||
color: var(--fs-accent);
|
color: var(--fs-accent);
|
||||||
border-color: var(--fs-accent);
|
border-color: var(--fs-accent);
|
||||||
}
|
}
|
||||||
|
/* A shipped default that moved between releases (#4225). Muted rather than
|
||||||
|
accented: it is the answer to "did I do this?" being NO for both of the
|
||||||
|
other two, and it wants to be legible without competing with the changes
|
||||||
|
somebody actually made. */
|
||||||
|
.tuning-actor.is-release {
|
||||||
|
color: var(--fs-text-secondary);
|
||||||
|
border-style: dashed;
|
||||||
|
}
|
||||||
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
|
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
|
||||||
.tuning-reason {
|
.tuning-reason {
|
||||||
margin: var(--fs-space-2) 0 0;
|
margin: var(--fs-space-2) 0 0;
|
||||||
|
|||||||
@@ -182,6 +182,33 @@ def create_app() -> Quart:
|
|||||||
start_notification_loop()
|
start_notification_loop()
|
||||||
start_auth_token_retention_loop()
|
start_auth_token_retention_loop()
|
||||||
|
|
||||||
|
# Write down any shipped default that moved in this release (#4225).
|
||||||
|
#
|
||||||
|
# INLINE AND AWAITED, deliberately, against the instinct the block
|
||||||
|
# below argues for. What #4181 cost three hours was CONCURRENCY — a
|
||||||
|
# background task created here racing this hook for the same pool.
|
||||||
|
# Running sequentially creates no such contention, and this is twelve
|
||||||
|
# single-row reads on an index against a table with a handful of rows.
|
||||||
|
#
|
||||||
|
# It has to finish before serving because the thing it protects is a
|
||||||
|
# telemetry read: `band_hugs_floor` compares a band against a floor,
|
||||||
|
# and across a release that changed the floor those describe different
|
||||||
|
# regimes. A readout served before the change was recorded is the very
|
||||||
|
# answer this exists to stop giving. Failure is logged and swallowed —
|
||||||
|
# provenance is worth a lot and never worth refusing to boot.
|
||||||
|
try:
|
||||||
|
from scribe.services.retrieval_tuning import record_release_defaults
|
||||||
|
moved = await record_release_defaults()
|
||||||
|
for row in moved:
|
||||||
|
if not row["baseline"]:
|
||||||
|
app.logger.info(
|
||||||
|
"release moved %s's shipped %s: %s -> %s",
|
||||||
|
row["surface"], row["dial"],
|
||||||
|
row["old_value"], row["new_value"],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
app.logger.warning("could not record release defaults", exc_info=True)
|
||||||
|
|
||||||
# Backfill embeddings for any notes that don't have one. Runs in the
|
# Backfill embeddings for any notes that don't have one. Runs in the
|
||||||
# background so it never blocks the server from accepting requests —
|
# background so it never blocks the server from accepting requests —
|
||||||
# and, since #4181, not until the rest of this hook has finished.
|
# and, since #4181, not until the rest of this hook has finished.
|
||||||
|
|||||||
@@ -425,6 +425,12 @@ It is an UPPER BOUND per surface: a pull records the door it came
|
|||||||
- `band_hugs_floor` — the weakest tenth of what an arm returns sits on
|
- `band_hugs_floor` — the weakest tenth of what an arm returns sits on
|
||||||
its floor. The bar is doing the selecting and the score is not, so
|
its floor. The bar is doing the selecting and the score is not, so
|
||||||
moving that floor changes how MUCH you get, not how good it is.
|
moving that floor changes how MUCH you get, not how good it is.
|
||||||
|
- `floor_moved_mid_window` — that arm's floor CHANGED inside the window,
|
||||||
|
by a release or by a dial turn, so its calls were made under two bars
|
||||||
|
and the band check above is suspended for it rather than answered
|
||||||
|
wrongly. Ask again with a `days` starting after the named date. The
|
||||||
|
warning replaces `band_hugs_floor` for that arm; it never accompanies
|
||||||
|
it.
|
||||||
- `no_duration` — rows written without timings. A logging gap, not a slow
|
- `no_duration` — rows written without timings. A logging gap, not a slow
|
||||||
arm, and it devalues every other number from that source.
|
arm, and it devalues every other number from that source.
|
||||||
- `surfaced_never_pulled` — distinct records shown and never opened, per
|
- `surfaced_never_pulled` — distinct records shown and never opened, per
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ from scribe.services.retrieval_registry import (
|
|||||||
POINTS, UNBIDDEN, get_point, is_registered, sources_expected_to_emit,
|
POINTS, UNBIDDEN, get_point, is_registered, sources_expected_to_emit,
|
||||||
)
|
)
|
||||||
from scribe.services.retrieval_surfaces import SURFACES, floor_for
|
from scribe.services.retrieval_surfaces import SURFACES, floor_for
|
||||||
|
from scribe.services.retrieval_tuning import floor_moves_since
|
||||||
from scribe.services.settings import get_setting
|
from scribe.services.settings import get_setting
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -498,7 +499,8 @@ def _warn(code, detail, source=None, **numbers) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
||||||
floors: dict, min_calls: int, epsilon: float) -> list[dict]:
|
floors: dict, min_calls: int, epsilon: float,
|
||||||
|
floor_moves: dict | None = None) -> list[dict]:
|
||||||
"""The four checks, over whatever sources the window actually contains.
|
"""The four checks, over whatever sources the window actually contains.
|
||||||
|
|
||||||
DELIBERATELY NOT KEYED ON A HARD-CODED SOURCE LIST. An arm added next
|
DELIBERATELY NOT KEYED ON A HARD-CODED SOURCE LIST. An arm added next
|
||||||
@@ -581,9 +583,41 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
|||||||
# invite tuning a dial that does not exist.
|
# invite tuning a dial that does not exist.
|
||||||
floor = floors.get(name)
|
floor = floors.get(name)
|
||||||
p10 = (b.get("top_score") or {}).get("p10")
|
p10 = (b.get("top_score") or {}).get("p10")
|
||||||
|
moved = (floor_moves or {}).get(name)
|
||||||
if calls >= min_calls and floor is not None and p10 is not None:
|
if calls >= min_calls and floor is not None and p10 is not None:
|
||||||
|
# ── The floor moved inside the window ────────────────────────
|
||||||
|
#
|
||||||
|
# THE CHECK IS SUSPENDED, NOT SOFTENED, and this is the whole of
|
||||||
|
# #4225. `band_hugs_floor` asks whether the scores are piled on
|
||||||
|
# the bar. That question needs the scores and the bar to come from
|
||||||
|
# the same regime; across a floor change they do not, and the
|
||||||
|
# comparison quietly becomes one between two populations.
|
||||||
|
#
|
||||||
|
# It announced itself when the change was a RAISE: p10 computed
|
||||||
|
# over calls made under the old, lower bar came out BELOW today's
|
||||||
|
# floor, and the warning reported a band "-0.0216 above" its
|
||||||
|
# floor. A negative distance above something is not a number
|
||||||
|
# anybody can act on. A LOWERED floor hides better — the gap comes
|
||||||
|
# out comfortably positive and reads as a clean bill of health on
|
||||||
|
# a sample that half predates the bar being judged.
|
||||||
|
#
|
||||||
|
# So the honest move is to say the sample cannot answer, and say
|
||||||
|
# when it will be able to, rather than print a figure with an
|
||||||
|
# asterisk. A reader who is told a number is unavailable goes and
|
||||||
|
# gets one; a reader handed a qualified number uses it.
|
||||||
|
if moved is not None:
|
||||||
|
out.append(_warn(
|
||||||
|
"floor_moved_mid_window",
|
||||||
|
f"this arm's floor changed at {moved}, inside the window, "
|
||||||
|
f"so its calls were made under two different bars and its "
|
||||||
|
f"band cannot be compared against the floor now in force "
|
||||||
|
f"({floor}). The band check is suspended for this arm "
|
||||||
|
f"until the window clears that date — ask again with a "
|
||||||
|
f"`days` that starts after it, or wait.",
|
||||||
|
source=name, floor=floor, p10=p10, moved_at=moved,
|
||||||
|
))
|
||||||
|
elif p10 - floor < epsilon:
|
||||||
gap = p10 - floor
|
gap = p10 - floor
|
||||||
if gap < epsilon:
|
|
||||||
out.append(_warn(
|
out.append(_warn(
|
||||||
"band_hugs_floor",
|
"band_hugs_floor",
|
||||||
f"the weakest tenth of what this arm returns scores "
|
f"the weakest tenth of what this arm returns scores "
|
||||||
@@ -1350,8 +1384,20 @@ async def retrieval_summary(
|
|||||||
except Exception: # pragma: no cover - telemetry never raises
|
except Exception: # pragma: no cover - telemetry never raises
|
||||||
logger.warning("could not read floor for %s", name, exc_info=True)
|
logger.warning("could not read floor for %s", name, exc_info=True)
|
||||||
|
|
||||||
|
# Floors that MOVED inside this window (#4225) — a release's shipped
|
||||||
|
# default or the operator's own dial, because the question is about the
|
||||||
|
# sample, not about who is answerable for it. Read unconditionally rather
|
||||||
|
# than under `if user_id`, unlike `floors` above: a release change belongs
|
||||||
|
# to no account, and it is exactly the case that used to go unrecorded.
|
||||||
|
floor_moves: dict[str, str] = {}
|
||||||
|
try:
|
||||||
|
floor_moves = await floor_moves_since(since)
|
||||||
|
except Exception: # pragma: no cover - telemetry never raises
|
||||||
|
logger.warning("could not read floor changes", exc_info=True)
|
||||||
|
|
||||||
out["warnings"] = _compute_warnings(
|
out["warnings"] = _compute_warnings(
|
||||||
out["sources"], usage, rule_usage, floors, min_calls, epsilon,
|
out["sources"], usage, rule_usage, floors, min_calls, epsilon,
|
||||||
|
floor_moves,
|
||||||
)
|
)
|
||||||
|
|
||||||
# "Active" means this window saw real traffic SOMEWHERE. Without that
|
# "Active" means this window saw real traffic SOMEWHERE. Without that
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
|
|
||||||
from scribe.models import async_session
|
from scribe.models import async_session
|
||||||
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
||||||
@@ -61,6 +61,10 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
DIALS = ("floor", "budget")
|
DIALS = ("floor", "budget")
|
||||||
|
|
||||||
|
# The actor for a change nobody made by hand: the shipped default moved
|
||||||
|
# between releases. Beside "model" and "human" (#4225).
|
||||||
|
RELEASE_ACTOR = "release"
|
||||||
|
|
||||||
# Long enough to say what was read and what it showed; short enough that nobody
|
# Long enough to say what was read and what it showed; short enough that nobody
|
||||||
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
||||||
# at which "0.66" stops being an acceptable answer to "why".
|
# at which "0.66" stops being an acceptable answer to "why".
|
||||||
@@ -168,7 +172,21 @@ async def current_settings(user_id: int) -> list[dict]:
|
|||||||
select(RetrievalTuningEvent)
|
select(RetrievalTuningEvent)
|
||||||
.where(
|
.where(
|
||||||
RetrievalTuningEvent.surface == name,
|
RetrievalTuningEvent.surface == name,
|
||||||
|
# THIS USER'S CHANGES *OR* A RELEASE'S (#4225).
|
||||||
|
# A release's rows carry no user id because no user
|
||||||
|
# made them, and taking the newest of the two is
|
||||||
|
# what makes the answer true: a dial the operator
|
||||||
|
# has tuned is explained by their change, and an
|
||||||
|
# untouched one by the release that last shipped
|
||||||
|
# its default. Filtering to the user alone reported
|
||||||
|
# "still on the shipped starting point" for a
|
||||||
|
# default that had in fact moved — the one state a
|
||||||
|
# reader would not think to check, which is the
|
||||||
|
# same sentence #4104 wrote about the bug above.
|
||||||
|
or_(
|
||||||
RetrievalTuningEvent.user_id == user_id,
|
RetrievalTuningEvent.user_id == user_id,
|
||||||
|
RetrievalTuningEvent.user_id.is_(None),
|
||||||
|
),
|
||||||
RetrievalTuningEvent.dial == dial,
|
RetrievalTuningEvent.dial == dial,
|
||||||
)
|
)
|
||||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
@@ -282,6 +300,126 @@ async def set_dial(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def record_release_defaults() -> list[dict]:
|
||||||
|
"""Write down that a RELEASE moved a shipped default (#4225).
|
||||||
|
|
||||||
|
THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a
|
||||||
|
person or a model choosing a number. It is silent about the other way a
|
||||||
|
floor moves, which is somebody editing `floor_default` in the registry and
|
||||||
|
shipping it. That change is invisible to every consumer of this table, and
|
||||||
|
one of those consumers is `band_hugs_floor`, which compares a percentile
|
||||||
|
against a floor and has no way to notice they come from different regimes.
|
||||||
|
|
||||||
|
Measured on the instance that found this: `write_path_rule` went 0.68 ->
|
||||||
|
0.72 on 2026-09-02 as a shipped default, and a 30-day window opening
|
||||||
|
2026-08-22 therefore held six days of calls made under the old bar. The
|
||||||
|
warning reported the band sitting "-0.0216 above" its floor — a negative
|
||||||
|
distance above something, which is what a two-population comparison looks
|
||||||
|
like when it finally says so out loud.
|
||||||
|
|
||||||
|
`user_id IS NULL`, because no user did this. A release acts on every
|
||||||
|
account that has not overridden the dial, and writing one row per user
|
||||||
|
would both multiply the row and misattribute it. Readers take the newest of
|
||||||
|
(this user's own change, the release's) — a dial the operator has tuned is
|
||||||
|
explained by their change, and an untouched one by the release.
|
||||||
|
|
||||||
|
`actor="release"`, a third value beside "model" and "human". The column is
|
||||||
|
Text with no CHECK precisely so a new kind of actor is not a migration —
|
||||||
|
the model's own comment says so, and this is the case it anticipated.
|
||||||
|
|
||||||
|
THE FIRST SIGHTING IS A BASELINE, NOT A CHANGE, and is written with
|
||||||
|
`old_value=None`. Nothing moved; the row exists so that the NEXT release
|
||||||
|
has something to be different from. That distinction is load-bearing
|
||||||
|
downstream: a warning suspends itself on a genuine move and must not
|
||||||
|
suspend itself on a fresh install's baseline, and it tells the two apart by
|
||||||
|
exactly this null.
|
||||||
|
|
||||||
|
Idempotent, and safe to call on every boot: a default that matches the last
|
||||||
|
recorded one writes nothing.
|
||||||
|
"""
|
||||||
|
written: list[dict] = []
|
||||||
|
stamp = calibration_stamp()
|
||||||
|
async with async_session() as session:
|
||||||
|
for name in surface_names():
|
||||||
|
s = get_surface(name)
|
||||||
|
for dial, shipped in (
|
||||||
|
("floor", float(s.floor_default)),
|
||||||
|
("budget", float(s.budget_default)),
|
||||||
|
):
|
||||||
|
last = (
|
||||||
|
await session.execute(
|
||||||
|
select(RetrievalTuningEvent)
|
||||||
|
.where(
|
||||||
|
RetrievalTuningEvent.surface == name,
|
||||||
|
RetrievalTuningEvent.user_id.is_(None),
|
||||||
|
RetrievalTuningEvent.actor == RELEASE_ACTOR,
|
||||||
|
RetrievalTuningEvent.dial == dial,
|
||||||
|
)
|
||||||
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalars().first()
|
||||||
|
if last is not None and abs(float(last.new_value) - shipped) < 1e-9:
|
||||||
|
continue
|
||||||
|
old = None if last is None else float(last.new_value)
|
||||||
|
reason = (
|
||||||
|
f"Baseline: this release ships {name}'s {dial} at {shipped}. "
|
||||||
|
f"Recorded so a later change to the shipped value has a "
|
||||||
|
f"date and a predecessor to be measured against; nothing "
|
||||||
|
f"moved here."
|
||||||
|
if old is None else
|
||||||
|
f"A release moved {name}'s shipped {dial} from {old} to "
|
||||||
|
f"{shipped}. Not a dial turn — the default in the registry "
|
||||||
|
f"changed, so calls logged either side of this date were "
|
||||||
|
f"made under different bars and cannot be pooled."
|
||||||
|
)
|
||||||
|
session.add(RetrievalTuningEvent(
|
||||||
|
user_id=None, surface=name, dial=dial,
|
||||||
|
old_value=old, new_value=shipped,
|
||||||
|
actor=RELEASE_ACTOR, reason=reason,
|
||||||
|
embedding_model=stamp["embedding_model"],
|
||||||
|
shape_version=stamp["shape_version"],
|
||||||
|
))
|
||||||
|
written.append({
|
||||||
|
"surface": name, "dial": dial,
|
||||||
|
"old_value": old, "new_value": shipped,
|
||||||
|
"baseline": old is None,
|
||||||
|
})
|
||||||
|
if written:
|
||||||
|
await session.commit()
|
||||||
|
return written
|
||||||
|
|
||||||
|
|
||||||
|
async def floor_moves_since(since) -> dict[str, str]:
|
||||||
|
"""Surfaces whose floor genuinely MOVED at or after `since`.
|
||||||
|
|
||||||
|
Genuinely: `old_value IS NOT NULL`, so a baseline row — which records a
|
||||||
|
default without changing it — does not read as a change. Returns surface ->
|
||||||
|
ISO timestamp of the newest such move, for a reader deciding whether a
|
||||||
|
window's numbers can be pooled.
|
||||||
|
|
||||||
|
Covers both kinds of move, a release's and this user's, because the
|
||||||
|
question is about the SAMPLE rather than about who is responsible for it:
|
||||||
|
a floor that moved mid-window splits the calls either way round.
|
||||||
|
"""
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
async with async_session() as session:
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(RetrievalTuningEvent)
|
||||||
|
.where(
|
||||||
|
RetrievalTuningEvent.dial == "floor",
|
||||||
|
RetrievalTuningEvent.old_value.isnot(None),
|
||||||
|
RetrievalTuningEvent.created_at >= since,
|
||||||
|
)
|
||||||
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
for row in rows:
|
||||||
|
out.setdefault(row.surface, row.created_at.isoformat())
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def tuning_history(
|
async def tuning_history(
|
||||||
user_id: int, *, surface: str | None = None, limit: int = 20
|
user_id: int, *, surface: str | None = None, limit: int = 20
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
@@ -296,7 +434,13 @@ async def tuning_history(
|
|||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
q = (
|
q = (
|
||||||
select(RetrievalTuningEvent)
|
select(RetrievalTuningEvent)
|
||||||
.where(RetrievalTuningEvent.user_id == user_id)
|
.where(or_(
|
||||||
|
RetrievalTuningEvent.user_id == user_id,
|
||||||
|
# Shipped-default changes, which belong to no account and
|
||||||
|
# would otherwise be missing from the one surface built
|
||||||
|
# for asking why a number is where it is (#4225).
|
||||||
|
RetrievalTuningEvent.user_id.is_(None),
|
||||||
|
))
|
||||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||||
.limit(max(1, min(int(limit), 200)))
|
.limit(max(1, min(int(limit), 200)))
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -196,3 +196,147 @@ def test_every_tool_in_the_module_is_registered():
|
|||||||
"retrieval_surfaces", "migrate_retrieval_floor",
|
"retrieval_surfaces", "migrate_retrieval_floor",
|
||||||
"tune_retrieval", "retrieval_tuning_history",
|
"tune_retrieval", "retrieval_tuning_history",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ── record_release_defaults / floor_moves_since (#4225) ───────────────────
|
||||||
|
#
|
||||||
|
# THE HOLE THESE FILL. This table records dial turns — a person or a model
|
||||||
|
# choosing a number. It was silent about the other way a floor moves: somebody
|
||||||
|
# edits `floor_default` in the registry and ships it. That change is invisible
|
||||||
|
# to every consumer of this table, and one consumer is `band_hugs_floor`, which
|
||||||
|
# compares a band against a floor and could not notice they came from different
|
||||||
|
# regimes.
|
||||||
|
#
|
||||||
|
# Measured on the instance that found it: `write_path_rule` went 0.68 -> 0.72
|
||||||
|
# on 2026-09-02 as a shipped default, so a 30-day window opening 2026-08-22
|
||||||
|
# held six days of calls made under the old bar. The readout reported the band
|
||||||
|
# sitting "-0.0216 above" its floor — which is what a two-population comparison
|
||||||
|
# looks like when it finally says so out loud.
|
||||||
|
|
||||||
|
|
||||||
|
def _release_rows(rows):
|
||||||
|
"""Patch `async_session` so the recorder sees `rows` as what is on record.
|
||||||
|
|
||||||
|
`rows` maps (surface, dial) -> new_value already recorded by a release.
|
||||||
|
"""
|
||||||
|
session = make_mock_session()
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
async def execute(stmt):
|
||||||
|
seen.append(stmt)
|
||||||
|
result = MagicMock()
|
||||||
|
# The recorder asks one question at a time, in registry order, so the
|
||||||
|
# answers are handed back in the order it asks them.
|
||||||
|
key = seen_keys.pop(0) if seen_keys else None
|
||||||
|
row = None
|
||||||
|
if key is not None and key in rows:
|
||||||
|
row = MagicMock()
|
||||||
|
row.new_value = rows[key]
|
||||||
|
result.scalars.return_value.first.return_value = row
|
||||||
|
return result
|
||||||
|
|
||||||
|
seen_keys = [(n, d) for n in rt.surface_names() for d in ("floor", "budget")]
|
||||||
|
session.execute = AsyncMock(side_effect=execute)
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_first_boot_records_a_baseline_and_calls_it_one():
|
||||||
|
"""Nothing moved. The row exists so the NEXT release has a predecessor.
|
||||||
|
|
||||||
|
Load-bearing downstream: the band check suspends itself on a genuine move
|
||||||
|
and must NOT suspend itself on a fresh install's baseline. It tells them
|
||||||
|
apart by `old_value` being null, so a baseline that claimed a change would
|
||||||
|
silently retire the check on every new install.
|
||||||
|
"""
|
||||||
|
session = _release_rows({})
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
written = await rt.record_release_defaults()
|
||||||
|
assert written, "a first boot records every dial"
|
||||||
|
assert all(w["baseline"] for w in written)
|
||||||
|
assert all(w["old_value"] is None for w in written)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_default_that_did_not_move_writes_nothing():
|
||||||
|
"""Called on every boot, so it has to be idempotent — otherwise the
|
||||||
|
history fills with rows saying the release shipped the same number again,
|
||||||
|
and a history nobody can skim is one nobody reads."""
|
||||||
|
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
|
||||||
|
else float(rt.get_surface(n).budget_default))
|
||||||
|
for n in rt.surface_names() for d in ("floor", "budget")}
|
||||||
|
session = _release_rows(current)
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
written = await rt.record_release_defaults()
|
||||||
|
assert written == []
|
||||||
|
session.add.assert_not_called()
|
||||||
|
session.commit.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_moved_default_is_recorded_with_both_values():
|
||||||
|
"""The event the whole task is about, and it carries what it moved FROM —
|
||||||
|
without that a reader knows a change happened and nothing about whether
|
||||||
|
the old sample can be pooled with the new one."""
|
||||||
|
name = rt.surface_names()[0]
|
||||||
|
s = rt.get_surface(name)
|
||||||
|
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
|
||||||
|
else float(rt.get_surface(n).budget_default))
|
||||||
|
for n in rt.surface_names() for d in ("floor", "budget")}
|
||||||
|
current[(name, "floor")] = s.floor_default - 0.04 # what the last release shipped
|
||||||
|
session = _release_rows(current)
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
written = await rt.record_release_defaults()
|
||||||
|
moved = [w for w in written if w["surface"] == name and w["dial"] == "floor"]
|
||||||
|
assert len(moved) == 1
|
||||||
|
assert moved[0]["baseline"] is False
|
||||||
|
assert moved[0]["old_value"] == pytest.approx(s.floor_default - 0.04)
|
||||||
|
assert moved[0]["new_value"] == pytest.approx(s.floor_default)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_release_row_belongs_to_no_account():
|
||||||
|
"""`user_id IS NULL`, because no user did this.
|
||||||
|
|
||||||
|
A release acts on every account that has not overridden the dial. Writing
|
||||||
|
one row per user would both multiply the row and misattribute it, and the
|
||||||
|
readers take the newest of (this user's change, the release's) — which only
|
||||||
|
works if the release's is distinguishable.
|
||||||
|
"""
|
||||||
|
session = _release_rows({})
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
await rt.record_release_defaults()
|
||||||
|
added = [c.args[0] for c in session.add.call_args_list]
|
||||||
|
assert added
|
||||||
|
assert all(e.user_id is None for e in added)
|
||||||
|
assert all(e.actor == rt.RELEASE_ACTOR for e in added)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_every_release_row_states_why_it_exists():
|
||||||
|
"""`reason` is the guardrail on this table, and a row written by machinery
|
||||||
|
is the one most likely to arrive blank."""
|
||||||
|
session = _release_rows({})
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
await rt.record_release_defaults()
|
||||||
|
added = [c.args[0] for c in session.add.call_args_list]
|
||||||
|
assert all(e.reason and e.reason.strip() for e in added)
|
||||||
|
assert all(e.surface in e.reason for e in added)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_baseline_is_not_reported_as_a_floor_move():
|
||||||
|
"""`floor_moves_since` is what suspends the band check, so it must ask for
|
||||||
|
a genuine move — `old_value IS NOT NULL` — rather than for any row."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
session = make_mock_session()
|
||||||
|
result = MagicMock()
|
||||||
|
result.scalars.return_value.all.return_value = []
|
||||||
|
session.execute = AsyncMock(return_value=result)
|
||||||
|
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||||
|
out = await rt.floor_moves_since(datetime.now(timezone.utc))
|
||||||
|
assert out == {}
|
||||||
|
# The filter is the whole correctness argument; assert it is in the query.
|
||||||
|
stmt = str(session.execute.call_args.args[0])
|
||||||
|
assert "old_value IS NOT NULL" in stmt
|
||||||
|
assert "dial" in stmt
|
||||||
|
|||||||
@@ -49,9 +49,10 @@ def src(**kw) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def warn(sources, usage=None, rule_usage=None, floors=None,
|
def warn(sources, usage=None, rule_usage=None, floors=None,
|
||||||
min_calls=N, epsilon=EPS) -> list[dict]:
|
min_calls=N, epsilon=EPS, floor_moves=None) -> list[dict]:
|
||||||
return _compute_warnings(
|
return _compute_warnings(
|
||||||
sources, usage or {}, rule_usage or {}, floors or {}, min_calls, epsilon,
|
sources, usage or {}, rule_usage or {}, floors or {}, min_calls, epsilon,
|
||||||
|
floor_moves or {},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -349,3 +350,99 @@ def test_an_absent_rule_usage_block_is_not_a_finding() -> None:
|
|||||||
assert "outcomes_never_recorded" not in codes(
|
assert "outcomes_never_recorded" not in codes(
|
||||||
warn({}, rule_usage={"rule_usage_failed": True})
|
warn({}, rule_usage={"rule_usage_failed": True})
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── floor_moved_mid_window (#4225) ────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# WHY THE BAND CHECK IS SUSPENDED RATHER THAN SOFTENED.
|
||||||
|
#
|
||||||
|
# `band_hugs_floor` asks whether the scores are piled on the bar. That needs
|
||||||
|
# the scores and the bar to come from the same regime, and across a floor
|
||||||
|
# change they do not — the comparison silently becomes one between two
|
||||||
|
# populations.
|
||||||
|
#
|
||||||
|
# It announced itself when the change was a RAISE: on the instance this was
|
||||||
|
# found on, `write_path_rule` went 0.68 -> 0.72 as a shipped default inside
|
||||||
|
# the window, and p10 computed over calls made under the old bar came out
|
||||||
|
# BELOW the new floor. The readout printed a band "-0.0216 above" its floor.
|
||||||
|
#
|
||||||
|
# A LOWERED floor is the dangerous one, because it hides: the gap comes out
|
||||||
|
# comfortably positive and reads as a clean bill of health on a sample that
|
||||||
|
# half predates the bar being judged. Both directions are pinned below.
|
||||||
|
|
||||||
|
MOVED = "2026-09-02T00:00:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_floor_that_moved_in_the_window_suspends_the_band_check() -> None:
|
||||||
|
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||||
|
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||||
|
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||||
|
assert "band_hugs_floor" not in codes(ws), (
|
||||||
|
"a suspended check must not also answer — the two never accompany "
|
||||||
|
"each other, or the reader gets a number and a warning about it"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_impossible_negative_gap_is_not_printed_at_all() -> None:
|
||||||
|
"""The symptom that exposed this: p10 BELOW the floor that gates the arm.
|
||||||
|
|
||||||
|
Arithmetically impossible inside one population, and the sentence built
|
||||||
|
from it ("only -0.0216 above") is not one anybody can act on.
|
||||||
|
"""
|
||||||
|
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.6984)},
|
||||||
|
floors={"auto_inject": 0.72}, floor_moves={"auto_inject": MOVED})
|
||||||
|
assert "band_hugs_floor" not in codes(ws)
|
||||||
|
assert not any(w.get("numbers", {}).get("gap", 0) < 0 for w in ws)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_lowered_floor_is_suspended_too_though_its_gap_looks_healthy() -> None:
|
||||||
|
"""The direction that does NOT announce itself.
|
||||||
|
|
||||||
|
A gap of 0.10 reads as a comfortable margin. It is computed over calls
|
||||||
|
half of which were made under a different bar, so it is not a margin at
|
||||||
|
all — and nothing in the number says so.
|
||||||
|
"""
|
||||||
|
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.80)},
|
||||||
|
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||||
|
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_floor_that_did_not_move_still_gets_judged() -> None:
|
||||||
|
"""The mirror error, and the expensive one: suspending on nothing would
|
||||||
|
retire a working check."""
|
||||||
|
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||||
|
floors={"auto_inject": 0.70}, floor_moves={})
|
||||||
|
assert "band_hugs_floor" in codes(ws, "auto_inject")
|
||||||
|
assert "floor_moved_mid_window" not in codes(ws)
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_the_arm_that_moved_is_suspended() -> None:
|
||||||
|
"""Surfaces are judged independently; one arm's release change says
|
||||||
|
nothing about another's sample."""
|
||||||
|
ws = warn(
|
||||||
|
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705),
|
||||||
|
"write_path": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||||
|
floors={"auto_inject": 0.70, "write_path": 0.70},
|
||||||
|
floor_moves={"auto_inject": MOVED},
|
||||||
|
)
|
||||||
|
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||||
|
assert "band_hugs_floor" in codes(ws, "write_path")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_warning_says_when_and_what_to_do_about_it() -> None:
|
||||||
|
"""A finding with no remedy is a complaint. The reader needs the date, so
|
||||||
|
they can ask again with a window that starts after it."""
|
||||||
|
w = next(w for w in warn(
|
||||||
|
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||||
|
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED},
|
||||||
|
) if w["code"] == "floor_moved_mid_window")
|
||||||
|
assert w["numbers"]["moved_at"] == MOVED
|
||||||
|
assert MOVED in w["detail"] and "days" in w["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_quiet_arm_is_not_suspended_either_way() -> None:
|
||||||
|
"""Below `min_calls` neither check runs — a moved floor does not promote
|
||||||
|
an arm nobody used into something worth a line."""
|
||||||
|
ws = warn({"auto_inject": src(calls=1, zero_result_calls=0, p10=0.705)},
|
||||||
|
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||||
|
assert "floor_moved_mid_window" not in codes(ws)
|
||||||
|
|||||||
Reference in New Issue
Block a user