Rule outcomes, the contract hint, and four extractor/backup fixes #174
@@ -253,6 +253,21 @@ async function loadTuningHistory() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Who moved a dial. Three answers, not two (#4225).
|
||||
*
|
||||
* `release` means the shipped default itself changed between versions — no
|
||||
* user turned anything. Folding that into the `human ? 'you' : 'Claude'`
|
||||
* fallback labelled it "Claude", which tells the operator the session moved a
|
||||
* floor it never touched. Misattribution is the one failure the `actor`
|
||||
* column exists to prevent, so an unknown value says so rather than guessing.
|
||||
*/
|
||||
function actorLabel(actor: string): string {
|
||||
if (actor === "human") return "you";
|
||||
if (actor === "release") return "this release";
|
||||
if (actor === "model") return "Claude";
|
||||
return actor;
|
||||
}
|
||||
|
||||
async function saveKbInject() {
|
||||
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
|
||||
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
|
||||
@@ -1925,8 +1940,11 @@ async function deleteUser(userId: number) {
|
||||
<template v-else>set to</template>
|
||||
{{ ev.new_value }}
|
||||
</span>
|
||||
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }">
|
||||
{{ ev.actor === 'human' ? 'you' : 'Claude' }}
|
||||
<span
|
||||
class="tuning-actor"
|
||||
:class="{ 'is-human': ev.actor === 'human', 'is-release': ev.actor === 'release' }"
|
||||
>
|
||||
{{ actorLabel(ev.actor) }}
|
||||
</span>
|
||||
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
|
||||
</div>
|
||||
@@ -4236,6 +4254,14 @@ async function deleteUser(userId: number) {
|
||||
color: var(--fs-accent);
|
||||
border-color: var(--fs-accent);
|
||||
}
|
||||
/* A shipped default that moved between releases (#4225). Muted rather than
|
||||
accented: it is the answer to "did I do this?" being NO for both of the
|
||||
other two, and it wants to be legible without competing with the changes
|
||||
somebody actually made. */
|
||||
.tuning-actor.is-release {
|
||||
color: var(--fs-text-secondary);
|
||||
border-style: dashed;
|
||||
}
|
||||
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
|
||||
.tuning-reason {
|
||||
margin: var(--fs-space-2) 0 0;
|
||||
|
||||
@@ -182,6 +182,33 @@ def create_app() -> Quart:
|
||||
start_notification_loop()
|
||||
start_auth_token_retention_loop()
|
||||
|
||||
# Write down any shipped default that moved in this release (#4225).
|
||||
#
|
||||
# INLINE AND AWAITED, deliberately, against the instinct the block
|
||||
# below argues for. What #4181 cost three hours was CONCURRENCY — a
|
||||
# background task created here racing this hook for the same pool.
|
||||
# Running sequentially creates no such contention, and this is twelve
|
||||
# single-row reads on an index against a table with a handful of rows.
|
||||
#
|
||||
# It has to finish before serving because the thing it protects is a
|
||||
# telemetry read: `band_hugs_floor` compares a band against a floor,
|
||||
# and across a release that changed the floor those describe different
|
||||
# regimes. A readout served before the change was recorded is the very
|
||||
# answer this exists to stop giving. Failure is logged and swallowed —
|
||||
# provenance is worth a lot and never worth refusing to boot.
|
||||
try:
|
||||
from scribe.services.retrieval_tuning import record_release_defaults
|
||||
moved = await record_release_defaults()
|
||||
for row in moved:
|
||||
if not row["baseline"]:
|
||||
app.logger.info(
|
||||
"release moved %s's shipped %s: %s -> %s",
|
||||
row["surface"], row["dial"],
|
||||
row["old_value"], row["new_value"],
|
||||
)
|
||||
except Exception:
|
||||
app.logger.warning("could not record release defaults", exc_info=True)
|
||||
|
||||
# Backfill embeddings for any notes that don't have one. Runs in the
|
||||
# background so it never blocks the server from accepting requests —
|
||||
# and, since #4181, not until the rest of this hook has finished.
|
||||
|
||||
@@ -425,6 +425,12 @@ It is an UPPER BOUND per surface: a pull records the door it came
|
||||
- `band_hugs_floor` — the weakest tenth of what an arm returns sits on
|
||||
its floor. The bar is doing the selecting and the score is not, so
|
||||
moving that floor changes how MUCH you get, not how good it is.
|
||||
- `floor_moved_mid_window` — that arm's floor CHANGED inside the window,
|
||||
by a release or by a dial turn, so its calls were made under two bars
|
||||
and the band check above is suspended for it rather than answered
|
||||
wrongly. Ask again with a `days` starting after the named date. The
|
||||
warning replaces `band_hugs_floor` for that arm; it never accompanies
|
||||
it.
|
||||
- `no_duration` — rows written without timings. A logging gap, not a slow
|
||||
arm, and it devalues every other number from that source.
|
||||
- `surfaced_never_pulled` — distinct records shown and never opened, per
|
||||
|
||||
@@ -40,6 +40,7 @@ from scribe.services.retrieval_registry import (
|
||||
POINTS, UNBIDDEN, get_point, is_registered, sources_expected_to_emit,
|
||||
)
|
||||
from scribe.services.retrieval_surfaces import SURFACES, floor_for
|
||||
from scribe.services.retrieval_tuning import floor_moves_since
|
||||
from scribe.services.settings import get_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -498,7 +499,8 @@ def _warn(code, detail, source=None, **numbers) -> dict:
|
||||
|
||||
|
||||
def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
||||
floors: dict, min_calls: int, epsilon: float) -> list[dict]:
|
||||
floors: dict, min_calls: int, epsilon: float,
|
||||
floor_moves: dict | None = None) -> list[dict]:
|
||||
"""The four checks, over whatever sources the window actually contains.
|
||||
|
||||
DELIBERATELY NOT KEYED ON A HARD-CODED SOURCE LIST. An arm added next
|
||||
@@ -581,9 +583,41 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
||||
# invite tuning a dial that does not exist.
|
||||
floor = floors.get(name)
|
||||
p10 = (b.get("top_score") or {}).get("p10")
|
||||
moved = (floor_moves or {}).get(name)
|
||||
if calls >= min_calls and floor is not None and p10 is not None:
|
||||
# ── The floor moved inside the window ────────────────────────
|
||||
#
|
||||
# THE CHECK IS SUSPENDED, NOT SOFTENED, and this is the whole of
|
||||
# #4225. `band_hugs_floor` asks whether the scores are piled on
|
||||
# the bar. That question needs the scores and the bar to come from
|
||||
# the same regime; across a floor change they do not, and the
|
||||
# comparison quietly becomes one between two populations.
|
||||
#
|
||||
# It announced itself when the change was a RAISE: p10 computed
|
||||
# over calls made under the old, lower bar came out BELOW today's
|
||||
# floor, and the warning reported a band "-0.0216 above" its
|
||||
# floor. A negative distance above something is not a number
|
||||
# anybody can act on. A LOWERED floor hides better — the gap comes
|
||||
# out comfortably positive and reads as a clean bill of health on
|
||||
# a sample that half predates the bar being judged.
|
||||
#
|
||||
# So the honest move is to say the sample cannot answer, and say
|
||||
# when it will be able to, rather than print a figure with an
|
||||
# asterisk. A reader who is told a number is unavailable goes and
|
||||
# gets one; a reader handed a qualified number uses it.
|
||||
if moved is not None:
|
||||
out.append(_warn(
|
||||
"floor_moved_mid_window",
|
||||
f"this arm's floor changed at {moved}, inside the window, "
|
||||
f"so its calls were made under two different bars and its "
|
||||
f"band cannot be compared against the floor now in force "
|
||||
f"({floor}). The band check is suspended for this arm "
|
||||
f"until the window clears that date — ask again with a "
|
||||
f"`days` that starts after it, or wait.",
|
||||
source=name, floor=floor, p10=p10, moved_at=moved,
|
||||
))
|
||||
elif p10 - floor < epsilon:
|
||||
gap = p10 - floor
|
||||
if gap < epsilon:
|
||||
out.append(_warn(
|
||||
"band_hugs_floor",
|
||||
f"the weakest tenth of what this arm returns scores "
|
||||
@@ -1350,8 +1384,20 @@ async def retrieval_summary(
|
||||
except Exception: # pragma: no cover - telemetry never raises
|
||||
logger.warning("could not read floor for %s", name, exc_info=True)
|
||||
|
||||
# Floors that MOVED inside this window (#4225) — a release's shipped
|
||||
# default or the operator's own dial, because the question is about the
|
||||
# sample, not about who is answerable for it. Read unconditionally rather
|
||||
# than under `if user_id`, unlike `floors` above: a release change belongs
|
||||
# to no account, and it is exactly the case that used to go unrecorded.
|
||||
floor_moves: dict[str, str] = {}
|
||||
try:
|
||||
floor_moves = await floor_moves_since(since)
|
||||
except Exception: # pragma: no cover - telemetry never raises
|
||||
logger.warning("could not read floor changes", exc_info=True)
|
||||
|
||||
out["warnings"] = _compute_warnings(
|
||||
out["sources"], usage, rule_usage, floors, min_calls, epsilon,
|
||||
floor_moves,
|
||||
)
|
||||
|
||||
# "Active" means this window saw real traffic SOMEWHERE. Without that
|
||||
|
||||
@@ -43,7 +43,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from scribe.models import async_session
|
||||
from scribe.models.retrieval_tuning import RetrievalTuningEvent
|
||||
@@ -61,6 +61,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
DIALS = ("floor", "budget")
|
||||
|
||||
# The actor for a change nobody made by hand: the shipped default moved
|
||||
# between releases. Beside "model" and "human" (#4225).
|
||||
RELEASE_ACTOR = "release"
|
||||
|
||||
# Long enough to say what was read and what it showed; short enough that nobody
|
||||
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
||||
# at which "0.66" stops being an acceptable answer to "why".
|
||||
@@ -168,7 +172,21 @@ async def current_settings(user_id: int) -> list[dict]:
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.surface == name,
|
||||
# THIS USER'S CHANGES *OR* A RELEASE'S (#4225).
|
||||
# A release's rows carry no user id because no user
|
||||
# made them, and taking the newest of the two is
|
||||
# what makes the answer true: a dial the operator
|
||||
# has tuned is explained by their change, and an
|
||||
# untouched one by the release that last shipped
|
||||
# its default. Filtering to the user alone reported
|
||||
# "still on the shipped starting point" for a
|
||||
# default that had in fact moved — the one state a
|
||||
# reader would not think to check, which is the
|
||||
# same sentence #4104 wrote about the bug above.
|
||||
or_(
|
||||
RetrievalTuningEvent.user_id == user_id,
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
),
|
||||
RetrievalTuningEvent.dial == dial,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
@@ -282,6 +300,126 @@ async def set_dial(
|
||||
}
|
||||
|
||||
|
||||
async def record_release_defaults() -> list[dict]:
|
||||
"""Write down that a RELEASE moved a shipped default (#4225).
|
||||
|
||||
THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a
|
||||
person or a model choosing a number. It is silent about the other way a
|
||||
floor moves, which is somebody editing `floor_default` in the registry and
|
||||
shipping it. That change is invisible to every consumer of this table, and
|
||||
one of those consumers is `band_hugs_floor`, which compares a percentile
|
||||
against a floor and has no way to notice they come from different regimes.
|
||||
|
||||
Measured on the instance that found this: `write_path_rule` went 0.68 ->
|
||||
0.72 on 2026-09-02 as a shipped default, and a 30-day window opening
|
||||
2026-08-22 therefore held six days of calls made under the old bar. The
|
||||
warning reported the band sitting "-0.0216 above" its floor — a negative
|
||||
distance above something, which is what a two-population comparison looks
|
||||
like when it finally says so out loud.
|
||||
|
||||
`user_id IS NULL`, because no user did this. A release acts on every
|
||||
account that has not overridden the dial, and writing one row per user
|
||||
would both multiply the row and misattribute it. Readers take the newest of
|
||||
(this user's own change, the release's) — a dial the operator has tuned is
|
||||
explained by their change, and an untouched one by the release.
|
||||
|
||||
`actor="release"`, a third value beside "model" and "human". The column is
|
||||
Text with no CHECK precisely so a new kind of actor is not a migration —
|
||||
the model's own comment says so, and this is the case it anticipated.
|
||||
|
||||
THE FIRST SIGHTING IS A BASELINE, NOT A CHANGE, and is written with
|
||||
`old_value=None`. Nothing moved; the row exists so that the NEXT release
|
||||
has something to be different from. That distinction is load-bearing
|
||||
downstream: a warning suspends itself on a genuine move and must not
|
||||
suspend itself on a fresh install's baseline, and it tells the two apart by
|
||||
exactly this null.
|
||||
|
||||
Idempotent, and safe to call on every boot: a default that matches the last
|
||||
recorded one writes nothing.
|
||||
"""
|
||||
written: list[dict] = []
|
||||
stamp = calibration_stamp()
|
||||
async with async_session() as session:
|
||||
for name in surface_names():
|
||||
s = get_surface(name)
|
||||
for dial, shipped in (
|
||||
("floor", float(s.floor_default)),
|
||||
("budget", float(s.budget_default)),
|
||||
):
|
||||
last = (
|
||||
await session.execute(
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.surface == name,
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
RetrievalTuningEvent.actor == RELEASE_ACTOR,
|
||||
RetrievalTuningEvent.dial == dial,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalars().first()
|
||||
if last is not None and abs(float(last.new_value) - shipped) < 1e-9:
|
||||
continue
|
||||
old = None if last is None else float(last.new_value)
|
||||
reason = (
|
||||
f"Baseline: this release ships {name}'s {dial} at {shipped}. "
|
||||
f"Recorded so a later change to the shipped value has a "
|
||||
f"date and a predecessor to be measured against; nothing "
|
||||
f"moved here."
|
||||
if old is None else
|
||||
f"A release moved {name}'s shipped {dial} from {old} to "
|
||||
f"{shipped}. Not a dial turn — the default in the registry "
|
||||
f"changed, so calls logged either side of this date were "
|
||||
f"made under different bars and cannot be pooled."
|
||||
)
|
||||
session.add(RetrievalTuningEvent(
|
||||
user_id=None, surface=name, dial=dial,
|
||||
old_value=old, new_value=shipped,
|
||||
actor=RELEASE_ACTOR, reason=reason,
|
||||
embedding_model=stamp["embedding_model"],
|
||||
shape_version=stamp["shape_version"],
|
||||
))
|
||||
written.append({
|
||||
"surface": name, "dial": dial,
|
||||
"old_value": old, "new_value": shipped,
|
||||
"baseline": old is None,
|
||||
})
|
||||
if written:
|
||||
await session.commit()
|
||||
return written
|
||||
|
||||
|
||||
async def floor_moves_since(since) -> dict[str, str]:
|
||||
"""Surfaces whose floor genuinely MOVED at or after `since`.
|
||||
|
||||
Genuinely: `old_value IS NOT NULL`, so a baseline row — which records a
|
||||
default without changing it — does not read as a change. Returns surface ->
|
||||
ISO timestamp of the newest such move, for a reader deciding whether a
|
||||
window's numbers can be pooled.
|
||||
|
||||
Covers both kinds of move, a release's and this user's, because the
|
||||
question is about the SAMPLE rather than about who is responsible for it:
|
||||
a floor that moved mid-window splits the calls either way round.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.dial == "floor",
|
||||
RetrievalTuningEvent.old_value.isnot(None),
|
||||
RetrievalTuningEvent.created_at >= since,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
)
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
out.setdefault(row.surface, row.created_at.isoformat())
|
||||
return out
|
||||
|
||||
|
||||
async def tuning_history(
|
||||
user_id: int, *, surface: str | None = None, limit: int = 20
|
||||
) -> list[dict]:
|
||||
@@ -296,7 +434,13 @@ async def tuning_history(
|
||||
async with async_session() as session:
|
||||
q = (
|
||||
select(RetrievalTuningEvent)
|
||||
.where(RetrievalTuningEvent.user_id == user_id)
|
||||
.where(or_(
|
||||
RetrievalTuningEvent.user_id == user_id,
|
||||
# Shipped-default changes, which belong to no account and
|
||||
# would otherwise be missing from the one surface built
|
||||
# for asking why a number is where it is (#4225).
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
))
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
.limit(max(1, min(int(limit), 200)))
|
||||
)
|
||||
|
||||
@@ -196,3 +196,147 @@ def test_every_tool_in_the_module_is_registered():
|
||||
"retrieval_surfaces", "migrate_retrieval_floor",
|
||||
"tune_retrieval", "retrieval_tuning_history",
|
||||
]
|
||||
|
||||
|
||||
# ── record_release_defaults / floor_moves_since (#4225) ───────────────────
|
||||
#
|
||||
# THE HOLE THESE FILL. This table records dial turns — a person or a model
|
||||
# choosing a number. It was silent about the other way a floor moves: somebody
|
||||
# edits `floor_default` in the registry and ships it. That change is invisible
|
||||
# to every consumer of this table, and one consumer is `band_hugs_floor`, which
|
||||
# compares a band against a floor and could not notice they came from different
|
||||
# regimes.
|
||||
#
|
||||
# Measured on the instance that found it: `write_path_rule` went 0.68 -> 0.72
|
||||
# on 2026-09-02 as a shipped default, so a 30-day window opening 2026-08-22
|
||||
# held six days of calls made under the old bar. The readout reported the band
|
||||
# sitting "-0.0216 above" its floor — which is what a two-population comparison
|
||||
# looks like when it finally says so out loud.
|
||||
|
||||
|
||||
def _release_rows(rows):
|
||||
"""Patch `async_session` so the recorder sees `rows` as what is on record.
|
||||
|
||||
`rows` maps (surface, dial) -> new_value already recorded by a release.
|
||||
"""
|
||||
session = make_mock_session()
|
||||
seen = []
|
||||
|
||||
async def execute(stmt):
|
||||
seen.append(stmt)
|
||||
result = MagicMock()
|
||||
# The recorder asks one question at a time, in registry order, so the
|
||||
# answers are handed back in the order it asks them.
|
||||
key = seen_keys.pop(0) if seen_keys else None
|
||||
row = None
|
||||
if key is not None and key in rows:
|
||||
row = MagicMock()
|
||||
row.new_value = rows[key]
|
||||
result.scalars.return_value.first.return_value = row
|
||||
return result
|
||||
|
||||
seen_keys = [(n, d) for n in rt.surface_names() for d in ("floor", "budget")]
|
||||
session.execute = AsyncMock(side_effect=execute)
|
||||
return session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_first_boot_records_a_baseline_and_calls_it_one():
|
||||
"""Nothing moved. The row exists so the NEXT release has a predecessor.
|
||||
|
||||
Load-bearing downstream: the band check suspends itself on a genuine move
|
||||
and must NOT suspend itself on a fresh install's baseline. It tells them
|
||||
apart by `old_value` being null, so a baseline that claimed a change would
|
||||
silently retire the check on every new install.
|
||||
"""
|
||||
session = _release_rows({})
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
written = await rt.record_release_defaults()
|
||||
assert written, "a first boot records every dial"
|
||||
assert all(w["baseline"] for w in written)
|
||||
assert all(w["old_value"] is None for w in written)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_default_that_did_not_move_writes_nothing():
|
||||
"""Called on every boot, so it has to be idempotent — otherwise the
|
||||
history fills with rows saying the release shipped the same number again,
|
||||
and a history nobody can skim is one nobody reads."""
|
||||
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
|
||||
else float(rt.get_surface(n).budget_default))
|
||||
for n in rt.surface_names() for d in ("floor", "budget")}
|
||||
session = _release_rows(current)
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
written = await rt.record_release_defaults()
|
||||
assert written == []
|
||||
session.add.assert_not_called()
|
||||
session.commit.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_moved_default_is_recorded_with_both_values():
|
||||
"""The event the whole task is about, and it carries what it moved FROM —
|
||||
without that a reader knows a change happened and nothing about whether
|
||||
the old sample can be pooled with the new one."""
|
||||
name = rt.surface_names()[0]
|
||||
s = rt.get_surface(name)
|
||||
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
|
||||
else float(rt.get_surface(n).budget_default))
|
||||
for n in rt.surface_names() for d in ("floor", "budget")}
|
||||
current[(name, "floor")] = s.floor_default - 0.04 # what the last release shipped
|
||||
session = _release_rows(current)
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
written = await rt.record_release_defaults()
|
||||
moved = [w for w in written if w["surface"] == name and w["dial"] == "floor"]
|
||||
assert len(moved) == 1
|
||||
assert moved[0]["baseline"] is False
|
||||
assert moved[0]["old_value"] == pytest.approx(s.floor_default - 0.04)
|
||||
assert moved[0]["new_value"] == pytest.approx(s.floor_default)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_release_row_belongs_to_no_account():
|
||||
"""`user_id IS NULL`, because no user did this.
|
||||
|
||||
A release acts on every account that has not overridden the dial. Writing
|
||||
one row per user would both multiply the row and misattribute it, and the
|
||||
readers take the newest of (this user's change, the release's) — which only
|
||||
works if the release's is distinguishable.
|
||||
"""
|
||||
session = _release_rows({})
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
await rt.record_release_defaults()
|
||||
added = [c.args[0] for c in session.add.call_args_list]
|
||||
assert added
|
||||
assert all(e.user_id is None for e in added)
|
||||
assert all(e.actor == rt.RELEASE_ACTOR for e in added)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_release_row_states_why_it_exists():
|
||||
"""`reason` is the guardrail on this table, and a row written by machinery
|
||||
is the one most likely to arrive blank."""
|
||||
session = _release_rows({})
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
await rt.record_release_defaults()
|
||||
added = [c.args[0] for c in session.add.call_args_list]
|
||||
assert all(e.reason and e.reason.strip() for e in added)
|
||||
assert all(e.surface in e.reason for e in added)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_baseline_is_not_reported_as_a_floor_move():
|
||||
"""`floor_moves_since` is what suspends the band check, so it must ask for
|
||||
a genuine move — `old_value IS NOT NULL` — rather than for any row."""
|
||||
from datetime import datetime, timezone
|
||||
session = make_mock_session()
|
||||
result = MagicMock()
|
||||
result.scalars.return_value.all.return_value = []
|
||||
session.execute = AsyncMock(return_value=result)
|
||||
with patch.object(rt, "async_session", MagicMock(return_value=session)):
|
||||
out = await rt.floor_moves_since(datetime.now(timezone.utc))
|
||||
assert out == {}
|
||||
# The filter is the whole correctness argument; assert it is in the query.
|
||||
stmt = str(session.execute.call_args.args[0])
|
||||
assert "old_value IS NOT NULL" in stmt
|
||||
assert "dial" in stmt
|
||||
|
||||
@@ -49,9 +49,10 @@ def src(**kw) -> dict:
|
||||
|
||||
|
||||
def warn(sources, usage=None, rule_usage=None, floors=None,
|
||||
min_calls=N, epsilon=EPS) -> list[dict]:
|
||||
min_calls=N, epsilon=EPS, floor_moves=None) -> list[dict]:
|
||||
return _compute_warnings(
|
||||
sources, usage or {}, rule_usage or {}, floors or {}, min_calls, epsilon,
|
||||
floor_moves or {},
|
||||
)
|
||||
|
||||
|
||||
@@ -349,3 +350,99 @@ def test_an_absent_rule_usage_block_is_not_a_finding() -> None:
|
||||
assert "outcomes_never_recorded" not in codes(
|
||||
warn({}, rule_usage={"rule_usage_failed": True})
|
||||
)
|
||||
|
||||
|
||||
# ── floor_moved_mid_window (#4225) ────────────────────────────────────────
|
||||
#
|
||||
# WHY THE BAND CHECK IS SUSPENDED RATHER THAN SOFTENED.
|
||||
#
|
||||
# `band_hugs_floor` asks whether the scores are piled on the bar. That needs
|
||||
# the scores and the bar to come from the same regime, and across a floor
|
||||
# change they do not — the comparison silently becomes one between two
|
||||
# populations.
|
||||
#
|
||||
# It announced itself when the change was a RAISE: on the instance this was
|
||||
# found on, `write_path_rule` went 0.68 -> 0.72 as a shipped default inside
|
||||
# the window, and p10 computed over calls made under the old bar came out
|
||||
# BELOW the new floor. The readout printed a band "-0.0216 above" its floor.
|
||||
#
|
||||
# A LOWERED floor is the dangerous one, because it hides: the gap comes out
|
||||
# comfortably positive and reads as a clean bill of health on a sample that
|
||||
# half predates the bar being judged. Both directions are pinned below.
|
||||
|
||||
MOVED = "2026-09-02T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_a_floor_that_moved_in_the_window_suspends_the_band_check() -> None:
|
||||
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||
assert "band_hugs_floor" not in codes(ws), (
|
||||
"a suspended check must not also answer — the two never accompany "
|
||||
"each other, or the reader gets a number and a warning about it"
|
||||
)
|
||||
|
||||
|
||||
def test_the_impossible_negative_gap_is_not_printed_at_all() -> None:
|
||||
"""The symptom that exposed this: p10 BELOW the floor that gates the arm.
|
||||
|
||||
Arithmetically impossible inside one population, and the sentence built
|
||||
from it ("only -0.0216 above") is not one anybody can act on.
|
||||
"""
|
||||
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.6984)},
|
||||
floors={"auto_inject": 0.72}, floor_moves={"auto_inject": MOVED})
|
||||
assert "band_hugs_floor" not in codes(ws)
|
||||
assert not any(w.get("numbers", {}).get("gap", 0) < 0 for w in ws)
|
||||
|
||||
|
||||
def test_a_lowered_floor_is_suspended_too_though_its_gap_looks_healthy() -> None:
|
||||
"""The direction that does NOT announce itself.
|
||||
|
||||
A gap of 0.10 reads as a comfortable margin. It is computed over calls
|
||||
half of which were made under a different bar, so it is not a margin at
|
||||
all — and nothing in the number says so.
|
||||
"""
|
||||
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.80)},
|
||||
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||
|
||||
|
||||
def test_a_floor_that_did_not_move_still_gets_judged() -> None:
|
||||
"""The mirror error, and the expensive one: suspending on nothing would
|
||||
retire a working check."""
|
||||
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||
floors={"auto_inject": 0.70}, floor_moves={})
|
||||
assert "band_hugs_floor" in codes(ws, "auto_inject")
|
||||
assert "floor_moved_mid_window" not in codes(ws)
|
||||
|
||||
|
||||
def test_only_the_arm_that_moved_is_suspended() -> None:
|
||||
"""Surfaces are judged independently; one arm's release change says
|
||||
nothing about another's sample."""
|
||||
ws = warn(
|
||||
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705),
|
||||
"write_path": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||
floors={"auto_inject": 0.70, "write_path": 0.70},
|
||||
floor_moves={"auto_inject": MOVED},
|
||||
)
|
||||
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
|
||||
assert "band_hugs_floor" in codes(ws, "write_path")
|
||||
|
||||
|
||||
def test_the_warning_says_when_and_what_to_do_about_it() -> None:
|
||||
"""A finding with no remedy is a complaint. The reader needs the date, so
|
||||
they can ask again with a window that starts after it."""
|
||||
w = next(w for w in warn(
|
||||
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
|
||||
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED},
|
||||
) if w["code"] == "floor_moved_mid_window")
|
||||
assert w["numbers"]["moved_at"] == MOVED
|
||||
assert MOVED in w["detail"] and "days" in w["detail"]
|
||||
|
||||
|
||||
def test_a_quiet_arm_is_not_suspended_either_way() -> None:
|
||||
"""Below `min_calls` neither check runs — a moved floor does not promote
|
||||
an arm nobody used into something worth a line."""
|
||||
ws = warn({"auto_inject": src(calls=1, zero_result_calls=0, p10=0.705)},
|
||||
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
|
||||
assert "floor_moved_mid_window" not in codes(ws)
|
||||
|
||||
Reference in New Issue
Block a user