Rule outcomes, the contract hint, and four extractor/backup fixes #174

Merged
bvandeusen merged 18 commits from dev into main 2026-09-21 06:31:10 -04:00
7 changed files with 499 additions and 9 deletions
Showing only changes of commit 512d0326a0 - Show all commits
+28 -2
View File
@@ -253,6 +253,21 @@ async function loadTuningHistory() {
} }
} }
/** Who moved a dial. Three answers, not two (#4225).
*
* `release` means the shipped default itself changed between versions — no
* user turned anything. Folding that into the `human ? 'you' : 'Claude'`
* fallback labelled it "Claude", which tells the operator the session moved a
* floor it never touched. Misattribution is the one failure the `actor`
* column exists to prevent, so an unknown value says so rather than guessing.
*/
function actorLabel(actor: string): string {
if (actor === "human") return "you";
if (actor === "release") return "this release";
if (actor === "model") return "Claude";
return actor;
}
async function saveKbInject() { async function saveKbInject() {
const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0)); const t = Math.min(1, Math.max(0, Number(kbInjectThreshold.value) || 0));
const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1))); const k = Math.min(10, Math.max(1, Math.floor(Number(kbInjectTopK.value) || 1)));
@@ -1925,8 +1940,11 @@ async function deleteUser(userId: number) {
<template v-else>set to</template> <template v-else>set to</template>
{{ ev.new_value }} {{ ev.new_value }}
</span> </span>
<span class="tuning-actor" :class="{ 'is-human': ev.actor === 'human' }"> <span
{{ ev.actor === 'human' ? 'you' : 'Claude' }} class="tuning-actor"
:class="{ 'is-human': ev.actor === 'human', 'is-release': ev.actor === 'release' }"
>
{{ actorLabel(ev.actor) }}
</span> </span>
<span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span> <span v-if="ev.created_at" class="tuning-when">{{ fmtDate(ev.created_at) }}</span>
</div> </div>
@@ -4236,6 +4254,14 @@ async function deleteUser(userId: number) {
color: var(--fs-accent); color: var(--fs-accent);
border-color: var(--fs-accent); border-color: var(--fs-accent);
} }
/* A shipped default that moved between releases (#4225). Muted rather than
accented: it is the answer to "did I do this?" being NO for both of the
other two, and it wants to be legible without competing with the changes
somebody actually made. */
.tuning-actor.is-release {
color: var(--fs-text-secondary);
border-style: dashed;
}
.tuning-when { margin-left: auto; color: var(--fs-text-tertiary); } .tuning-when { margin-left: auto; color: var(--fs-text-tertiary); }
.tuning-reason { .tuning-reason {
margin: var(--fs-space-2) 0 0; margin: var(--fs-space-2) 0 0;
+27
View File
@@ -182,6 +182,33 @@ def create_app() -> Quart:
start_notification_loop() start_notification_loop()
start_auth_token_retention_loop() start_auth_token_retention_loop()
# Write down any shipped default that moved in this release (#4225).
#
# INLINE AND AWAITED, deliberately, against the instinct the block
# below argues for. What #4181 cost three hours was CONCURRENCY — a
# background task created here racing this hook for the same pool.
# Running sequentially creates no such contention, and this is twelve
# single-row reads on an index against a table with a handful of rows.
#
# It has to finish before serving because the thing it protects is a
# telemetry read: `band_hugs_floor` compares a band against a floor,
# and across a release that changed the floor those describe different
# regimes. A readout served before the change was recorded is the very
# answer this exists to stop giving. Failure is logged and swallowed —
# provenance is worth a lot and never worth refusing to boot.
try:
from scribe.services.retrieval_tuning import record_release_defaults
moved = await record_release_defaults()
for row in moved:
if not row["baseline"]:
app.logger.info(
"release moved %s's shipped %s: %s -> %s",
row["surface"], row["dial"],
row["old_value"], row["new_value"],
)
except Exception:
app.logger.warning("could not record release defaults", exc_info=True)
# Backfill embeddings for any notes that don't have one. Runs in the # Backfill embeddings for any notes that don't have one. Runs in the
# background so it never blocks the server from accepting requests — # background so it never blocks the server from accepting requests —
# and, since #4181, not until the rest of this hook has finished. # and, since #4181, not until the rest of this hook has finished.
+6
View File
@@ -425,6 +425,12 @@ It is an UPPER BOUND per surface: a pull records the door it came
- `band_hugs_floor` — the weakest tenth of what an arm returns sits on - `band_hugs_floor` — the weakest tenth of what an arm returns sits on
its floor. The bar is doing the selecting and the score is not, so its floor. The bar is doing the selecting and the score is not, so
moving that floor changes how MUCH you get, not how good it is. moving that floor changes how MUCH you get, not how good it is.
- `floor_moved_mid_window` — that arm's floor CHANGED inside the window,
by a release or by a dial turn, so its calls were made under two bars
and the band check above is suspended for it rather than answered
wrongly. Ask again with a `days` starting after the named date. The
warning replaces `band_hugs_floor` for that arm; it never accompanies
it.
- `no_duration` — rows written without timings. A logging gap, not a slow - `no_duration` — rows written without timings. A logging gap, not a slow
arm, and it devalues every other number from that source. arm, and it devalues every other number from that source.
- `surfaced_never_pulled` — distinct records shown and never opened, per - `surfaced_never_pulled` — distinct records shown and never opened, per
+48 -2
View File
@@ -40,6 +40,7 @@ from scribe.services.retrieval_registry import (
POINTS, UNBIDDEN, get_point, is_registered, sources_expected_to_emit, POINTS, UNBIDDEN, get_point, is_registered, sources_expected_to_emit,
) )
from scribe.services.retrieval_surfaces import SURFACES, floor_for from scribe.services.retrieval_surfaces import SURFACES, floor_for
from scribe.services.retrieval_tuning import floor_moves_since
from scribe.services.settings import get_setting from scribe.services.settings import get_setting
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -498,7 +499,8 @@ def _warn(code, detail, source=None, **numbers) -> dict:
def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
floors: dict, min_calls: int, epsilon: float) -> list[dict]: floors: dict, min_calls: int, epsilon: float,
floor_moves: dict | None = None) -> list[dict]:
"""The four checks, over whatever sources the window actually contains. """The four checks, over whatever sources the window actually contains.
DELIBERATELY NOT KEYED ON A HARD-CODED SOURCE LIST. An arm added next DELIBERATELY NOT KEYED ON A HARD-CODED SOURCE LIST. An arm added next
@@ -581,9 +583,41 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
# invite tuning a dial that does not exist. # invite tuning a dial that does not exist.
floor = floors.get(name) floor = floors.get(name)
p10 = (b.get("top_score") or {}).get("p10") p10 = (b.get("top_score") or {}).get("p10")
moved = (floor_moves or {}).get(name)
if calls >= min_calls and floor is not None and p10 is not None: if calls >= min_calls and floor is not None and p10 is not None:
# ── The floor moved inside the window ────────────────────────
#
# THE CHECK IS SUSPENDED, NOT SOFTENED, and this is the whole of
# #4225. `band_hugs_floor` asks whether the scores are piled on
# the bar. That question needs the scores and the bar to come from
# the same regime; across a floor change they do not, and the
# comparison quietly becomes one between two populations.
#
# It announced itself when the change was a RAISE: p10 computed
# over calls made under the old, lower bar came out BELOW today's
# floor, and the warning reported a band "-0.0216 above" its
# floor. A negative distance above something is not a number
# anybody can act on. A LOWERED floor hides better — the gap comes
# out comfortably positive and reads as a clean bill of health on
# a sample that half predates the bar being judged.
#
# So the honest move is to say the sample cannot answer, and say
# when it will be able to, rather than print a figure with an
# asterisk. A reader who is told a number is unavailable goes and
# gets one; a reader handed a qualified number uses it.
if moved is not None:
out.append(_warn(
"floor_moved_mid_window",
f"this arm's floor changed at {moved}, inside the window, "
f"so its calls were made under two different bars and its "
f"band cannot be compared against the floor now in force "
f"({floor}). The band check is suspended for this arm "
f"until the window clears that date — ask again with a "
f"`days` that starts after it, or wait.",
source=name, floor=floor, p10=p10, moved_at=moved,
))
elif p10 - floor < epsilon:
gap = p10 - floor gap = p10 - floor
if gap < epsilon:
out.append(_warn( out.append(_warn(
"band_hugs_floor", "band_hugs_floor",
f"the weakest tenth of what this arm returns scores " f"the weakest tenth of what this arm returns scores "
@@ -1350,8 +1384,20 @@ async def retrieval_summary(
except Exception: # pragma: no cover - telemetry never raises except Exception: # pragma: no cover - telemetry never raises
logger.warning("could not read floor for %s", name, exc_info=True) logger.warning("could not read floor for %s", name, exc_info=True)
# Floors that MOVED inside this window (#4225) — a release's shipped
# default or the operator's own dial, because the question is about the
# sample, not about who is answerable for it. Read unconditionally rather
# than under `if user_id`, unlike `floors` above: a release change belongs
# to no account, and it is exactly the case that used to go unrecorded.
floor_moves: dict[str, str] = {}
try:
floor_moves = await floor_moves_since(since)
except Exception: # pragma: no cover - telemetry never raises
logger.warning("could not read floor changes", exc_info=True)
out["warnings"] = _compute_warnings( out["warnings"] = _compute_warnings(
out["sources"], usage, rule_usage, floors, min_calls, epsilon, out["sources"], usage, rule_usage, floors, min_calls, epsilon,
floor_moves,
) )
# "Active" means this window saw real traffic SOMEWHERE. Without that # "Active" means this window saw real traffic SOMEWHERE. Without that
+146 -2
View File
@@ -43,7 +43,7 @@ from __future__ import annotations
import logging import logging
from sqlalchemy import select from sqlalchemy import or_, select
from scribe.models import async_session from scribe.models import async_session
from scribe.models.retrieval_tuning import RetrievalTuningEvent from scribe.models.retrieval_tuning import RetrievalTuningEvent
@@ -61,6 +61,10 @@ logger = logging.getLogger(__name__)
DIALS = ("floor", "budget") DIALS = ("floor", "budget")
# The actor for a change nobody made by hand: the shipped default moved
# between releases. Beside "model" and "human" (#4225).
RELEASE_ACTOR = "release"
# Long enough to say what was read and what it showed; short enough that nobody # Long enough to say what was read and what it showed; short enough that nobody
# pastes a telemetry dump in. The number is not a measurement — it is the point # pastes a telemetry dump in. The number is not a measurement — it is the point
# at which "0.66" stops being an acceptable answer to "why". # at which "0.66" stops being an acceptable answer to "why".
@@ -168,7 +172,21 @@ async def current_settings(user_id: int) -> list[dict]:
select(RetrievalTuningEvent) select(RetrievalTuningEvent)
.where( .where(
RetrievalTuningEvent.surface == name, RetrievalTuningEvent.surface == name,
# THIS USER'S CHANGES *OR* A RELEASE'S (#4225).
# A release's rows carry no user id because no user
# made them, and taking the newest of the two is
# what makes the answer true: a dial the operator
# has tuned is explained by their change, and an
# untouched one by the release that last shipped
# its default. Filtering to the user alone reported
# "still on the shipped starting point" for a
# default that had in fact moved — the one state a
# reader would not think to check, which is the
# same sentence #4104 wrote about the bug above.
or_(
RetrievalTuningEvent.user_id == user_id, RetrievalTuningEvent.user_id == user_id,
RetrievalTuningEvent.user_id.is_(None),
),
RetrievalTuningEvent.dial == dial, RetrievalTuningEvent.dial == dial,
) )
.order_by(RetrievalTuningEvent.created_at.desc()) .order_by(RetrievalTuningEvent.created_at.desc())
@@ -282,6 +300,126 @@ async def set_dial(
} }
async def record_release_defaults() -> list[dict]:
"""Write down that a RELEASE moved a shipped default (#4225).
THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a
person or a model choosing a number. It is silent about the other way a
floor moves, which is somebody editing `floor_default` in the registry and
shipping it. That change is invisible to every consumer of this table, and
one of those consumers is `band_hugs_floor`, which compares a percentile
against a floor and has no way to notice they come from different regimes.
Measured on the instance that found this: `write_path_rule` went 0.68 ->
0.72 on 2026-09-02 as a shipped default, and a 30-day window opening
2026-08-22 therefore held six days of calls made under the old bar. The
warning reported the band sitting "-0.0216 above" its floor — a negative
distance above something, which is what a two-population comparison looks
like when it finally says so out loud.
`user_id IS NULL`, because no user did this. A release acts on every
account that has not overridden the dial, and writing one row per user
would both multiply the row and misattribute it. Readers take the newest of
(this user's own change, the release's) — a dial the operator has tuned is
explained by their change, and an untouched one by the release.
`actor="release"`, a third value beside "model" and "human". The column is
Text with no CHECK precisely so a new kind of actor is not a migration —
the model's own comment says so, and this is the case it anticipated.
THE FIRST SIGHTING IS A BASELINE, NOT A CHANGE, and is written with
`old_value=None`. Nothing moved; the row exists so that the NEXT release
has something to be different from. That distinction is load-bearing
downstream: a warning suspends itself on a genuine move and must not
suspend itself on a fresh install's baseline, and it tells the two apart by
exactly this null.
Idempotent, and safe to call on every boot: a default that matches the last
recorded one writes nothing.
"""
written: list[dict] = []
stamp = calibration_stamp()
async with async_session() as session:
for name in surface_names():
s = get_surface(name)
for dial, shipped in (
("floor", float(s.floor_default)),
("budget", float(s.budget_default)),
):
last = (
await session.execute(
select(RetrievalTuningEvent)
.where(
RetrievalTuningEvent.surface == name,
RetrievalTuningEvent.user_id.is_(None),
RetrievalTuningEvent.actor == RELEASE_ACTOR,
RetrievalTuningEvent.dial == dial,
)
.order_by(RetrievalTuningEvent.created_at.desc())
.limit(1)
)
).scalars().first()
if last is not None and abs(float(last.new_value) - shipped) < 1e-9:
continue
old = None if last is None else float(last.new_value)
reason = (
f"Baseline: this release ships {name}'s {dial} at {shipped}. "
f"Recorded so a later change to the shipped value has a "
f"date and a predecessor to be measured against; nothing "
f"moved here."
if old is None else
f"A release moved {name}'s shipped {dial} from {old} to "
f"{shipped}. Not a dial turn — the default in the registry "
f"changed, so calls logged either side of this date were "
f"made under different bars and cannot be pooled."
)
session.add(RetrievalTuningEvent(
user_id=None, surface=name, dial=dial,
old_value=old, new_value=shipped,
actor=RELEASE_ACTOR, reason=reason,
embedding_model=stamp["embedding_model"],
shape_version=stamp["shape_version"],
))
written.append({
"surface": name, "dial": dial,
"old_value": old, "new_value": shipped,
"baseline": old is None,
})
if written:
await session.commit()
return written
async def floor_moves_since(since) -> dict[str, str]:
"""Surfaces whose floor genuinely MOVED at or after `since`.
Genuinely: `old_value IS NOT NULL`, so a baseline row — which records a
default without changing it — does not read as a change. Returns surface ->
ISO timestamp of the newest such move, for a reader deciding whether a
window's numbers can be pooled.
Covers both kinds of move, a release's and this user's, because the
question is about the SAMPLE rather than about who is responsible for it:
a floor that moved mid-window splits the calls either way round.
"""
out: dict[str, str] = {}
async with async_session() as session:
rows = (
await session.execute(
select(RetrievalTuningEvent)
.where(
RetrievalTuningEvent.dial == "floor",
RetrievalTuningEvent.old_value.isnot(None),
RetrievalTuningEvent.created_at >= since,
)
.order_by(RetrievalTuningEvent.created_at.desc())
)
).scalars().all()
for row in rows:
out.setdefault(row.surface, row.created_at.isoformat())
return out
async def tuning_history( async def tuning_history(
user_id: int, *, surface: str | None = None, limit: int = 20 user_id: int, *, surface: str | None = None, limit: int = 20
) -> list[dict]: ) -> list[dict]:
@@ -296,7 +434,13 @@ async def tuning_history(
async with async_session() as session: async with async_session() as session:
q = ( q = (
select(RetrievalTuningEvent) select(RetrievalTuningEvent)
.where(RetrievalTuningEvent.user_id == user_id) .where(or_(
RetrievalTuningEvent.user_id == user_id,
# Shipped-default changes, which belong to no account and
# would otherwise be missing from the one surface built
# for asking why a number is where it is (#4225).
RetrievalTuningEvent.user_id.is_(None),
))
.order_by(RetrievalTuningEvent.created_at.desc()) .order_by(RetrievalTuningEvent.created_at.desc())
.limit(max(1, min(int(limit), 200))) .limit(max(1, min(int(limit), 200)))
) )
+144
View File
@@ -196,3 +196,147 @@ def test_every_tool_in_the_module_is_registered():
"retrieval_surfaces", "migrate_retrieval_floor", "retrieval_surfaces", "migrate_retrieval_floor",
"tune_retrieval", "retrieval_tuning_history", "tune_retrieval", "retrieval_tuning_history",
] ]
# ── record_release_defaults / floor_moves_since (#4225) ───────────────────
#
# THE HOLE THESE FILL. This table records dial turns — a person or a model
# choosing a number. It was silent about the other way a floor moves: somebody
# edits `floor_default` in the registry and ships it. That change is invisible
# to every consumer of this table, and one consumer is `band_hugs_floor`, which
# compares a band against a floor and could not notice they came from different
# regimes.
#
# Measured on the instance that found it: `write_path_rule` went 0.68 -> 0.72
# on 2026-09-02 as a shipped default, so a 30-day window opening 2026-08-22
# held six days of calls made under the old bar. The readout reported the band
# sitting "-0.0216 above" its floor — which is what a two-population comparison
# looks like when it finally says so out loud.
def _release_rows(rows):
"""Patch `async_session` so the recorder sees `rows` as what is on record.
`rows` maps (surface, dial) -> new_value already recorded by a release.
"""
session = make_mock_session()
seen = []
async def execute(stmt):
seen.append(stmt)
result = MagicMock()
# The recorder asks one question at a time, in registry order, so the
# answers are handed back in the order it asks them.
key = seen_keys.pop(0) if seen_keys else None
row = None
if key is not None and key in rows:
row = MagicMock()
row.new_value = rows[key]
result.scalars.return_value.first.return_value = row
return result
seen_keys = [(n, d) for n in rt.surface_names() for d in ("floor", "budget")]
session.execute = AsyncMock(side_effect=execute)
return session
@pytest.mark.asyncio
async def test_a_first_boot_records_a_baseline_and_calls_it_one():
"""Nothing moved. The row exists so the NEXT release has a predecessor.
Load-bearing downstream: the band check suspends itself on a genuine move
and must NOT suspend itself on a fresh install's baseline. It tells them
apart by `old_value` being null, so a baseline that claimed a change would
silently retire the check on every new install.
"""
session = _release_rows({})
with patch.object(rt, "async_session", MagicMock(return_value=session)):
written = await rt.record_release_defaults()
assert written, "a first boot records every dial"
assert all(w["baseline"] for w in written)
assert all(w["old_value"] is None for w in written)
@pytest.mark.asyncio
async def test_a_default_that_did_not_move_writes_nothing():
"""Called on every boot, so it has to be idempotent — otherwise the
history fills with rows saying the release shipped the same number again,
and a history nobody can skim is one nobody reads."""
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
else float(rt.get_surface(n).budget_default))
for n in rt.surface_names() for d in ("floor", "budget")}
session = _release_rows(current)
with patch.object(rt, "async_session", MagicMock(return_value=session)):
written = await rt.record_release_defaults()
assert written == []
session.add.assert_not_called()
session.commit.assert_not_called()
@pytest.mark.asyncio
async def test_a_moved_default_is_recorded_with_both_values():
"""The event the whole task is about, and it carries what it moved FROM —
without that a reader knows a change happened and nothing about whether
the old sample can be pooled with the new one."""
name = rt.surface_names()[0]
s = rt.get_surface(name)
current = {(n, d): (rt.get_surface(n).floor_default if d == "floor"
else float(rt.get_surface(n).budget_default))
for n in rt.surface_names() for d in ("floor", "budget")}
current[(name, "floor")] = s.floor_default - 0.04 # what the last release shipped
session = _release_rows(current)
with patch.object(rt, "async_session", MagicMock(return_value=session)):
written = await rt.record_release_defaults()
moved = [w for w in written if w["surface"] == name and w["dial"] == "floor"]
assert len(moved) == 1
assert moved[0]["baseline"] is False
assert moved[0]["old_value"] == pytest.approx(s.floor_default - 0.04)
assert moved[0]["new_value"] == pytest.approx(s.floor_default)
@pytest.mark.asyncio
async def test_a_release_row_belongs_to_no_account():
"""`user_id IS NULL`, because no user did this.
A release acts on every account that has not overridden the dial. Writing
one row per user would both multiply the row and misattribute it, and the
readers take the newest of (this user's change, the release's) — which only
works if the release's is distinguishable.
"""
session = _release_rows({})
with patch.object(rt, "async_session", MagicMock(return_value=session)):
await rt.record_release_defaults()
added = [c.args[0] for c in session.add.call_args_list]
assert added
assert all(e.user_id is None for e in added)
assert all(e.actor == rt.RELEASE_ACTOR for e in added)
@pytest.mark.asyncio
async def test_every_release_row_states_why_it_exists():
"""`reason` is the guardrail on this table, and a row written by machinery
is the one most likely to arrive blank."""
session = _release_rows({})
with patch.object(rt, "async_session", MagicMock(return_value=session)):
await rt.record_release_defaults()
added = [c.args[0] for c in session.add.call_args_list]
assert all(e.reason and e.reason.strip() for e in added)
assert all(e.surface in e.reason for e in added)
@pytest.mark.asyncio
async def test_a_baseline_is_not_reported_as_a_floor_move():
"""`floor_moves_since` is what suspends the band check, so it must ask for
a genuine move — `old_value IS NOT NULL` — rather than for any row."""
from datetime import datetime, timezone
session = make_mock_session()
result = MagicMock()
result.scalars.return_value.all.return_value = []
session.execute = AsyncMock(return_value=result)
with patch.object(rt, "async_session", MagicMock(return_value=session)):
out = await rt.floor_moves_since(datetime.now(timezone.utc))
assert out == {}
# The filter is the whole correctness argument; assert it is in the query.
stmt = str(session.execute.call_args.args[0])
assert "old_value IS NOT NULL" in stmt
assert "dial" in stmt
+98 -1
View File
@@ -49,9 +49,10 @@ def src(**kw) -> dict:
def warn(sources, usage=None, rule_usage=None, floors=None, def warn(sources, usage=None, rule_usage=None, floors=None,
min_calls=N, epsilon=EPS) -> list[dict]: min_calls=N, epsilon=EPS, floor_moves=None) -> list[dict]:
return _compute_warnings( return _compute_warnings(
sources, usage or {}, rule_usage or {}, floors or {}, min_calls, epsilon, sources, usage or {}, rule_usage or {}, floors or {}, min_calls, epsilon,
floor_moves or {},
) )
@@ -349,3 +350,99 @@ def test_an_absent_rule_usage_block_is_not_a_finding() -> None:
assert "outcomes_never_recorded" not in codes( assert "outcomes_never_recorded" not in codes(
warn({}, rule_usage={"rule_usage_failed": True}) warn({}, rule_usage={"rule_usage_failed": True})
) )
# ── floor_moved_mid_window (#4225) ────────────────────────────────────────
#
# WHY THE BAND CHECK IS SUSPENDED RATHER THAN SOFTENED.
#
# `band_hugs_floor` asks whether the scores are piled on the bar. That needs
# the scores and the bar to come from the same regime, and across a floor
# change they do not — the comparison silently becomes one between two
# populations.
#
# It announced itself when the change was a RAISE: on the instance this was
# found on, `write_path_rule` went 0.68 -> 0.72 as a shipped default inside
# the window, and p10 computed over calls made under the old bar came out
# BELOW the new floor. The readout printed a band "-0.0216 above" its floor.
#
# A LOWERED floor is the dangerous one, because it hides: the gap comes out
# comfortably positive and reads as a clean bill of health on a sample that
# half predates the bar being judged. Both directions are pinned below.
MOVED = "2026-09-02T00:00:00+00:00"
def test_a_floor_that_moved_in_the_window_suspends_the_band_check() -> None:
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
assert "band_hugs_floor" not in codes(ws), (
"a suspended check must not also answer — the two never accompany "
"each other, or the reader gets a number and a warning about it"
)
def test_the_impossible_negative_gap_is_not_printed_at_all() -> None:
"""The symptom that exposed this: p10 BELOW the floor that gates the arm.
Arithmetically impossible inside one population, and the sentence built
from it ("only -0.0216 above") is not one anybody can act on.
"""
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.6984)},
floors={"auto_inject": 0.72}, floor_moves={"auto_inject": MOVED})
assert "band_hugs_floor" not in codes(ws)
assert not any(w.get("numbers", {}).get("gap", 0) < 0 for w in ws)
def test_a_lowered_floor_is_suspended_too_though_its_gap_looks_healthy() -> None:
"""The direction that does NOT announce itself.
A gap of 0.10 reads as a comfortable margin. It is computed over calls
half of which were made under a different bar, so it is not a margin at
all — and nothing in the number says so.
"""
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.80)},
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
def test_a_floor_that_did_not_move_still_gets_judged() -> None:
"""The mirror error, and the expensive one: suspending on nothing would
retire a working check."""
ws = warn({"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
floors={"auto_inject": 0.70}, floor_moves={})
assert "band_hugs_floor" in codes(ws, "auto_inject")
assert "floor_moved_mid_window" not in codes(ws)
def test_only_the_arm_that_moved_is_suspended() -> None:
"""Surfaces are judged independently; one arm's release change says
nothing about another's sample."""
ws = warn(
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705),
"write_path": src(calls=100, zero_result_calls=5, p10=0.705)},
floors={"auto_inject": 0.70, "write_path": 0.70},
floor_moves={"auto_inject": MOVED},
)
assert "floor_moved_mid_window" in codes(ws, "auto_inject")
assert "band_hugs_floor" in codes(ws, "write_path")
def test_the_warning_says_when_and_what_to_do_about_it() -> None:
"""A finding with no remedy is a complaint. The reader needs the date, so
they can ask again with a window that starts after it."""
w = next(w for w in warn(
{"auto_inject": src(calls=100, zero_result_calls=5, p10=0.705)},
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED},
) if w["code"] == "floor_moved_mid_window")
assert w["numbers"]["moved_at"] == MOVED
assert MOVED in w["detail"] and "days" in w["detail"]
def test_a_quiet_arm_is_not_suspended_either_way() -> None:
"""Below `min_calls` neither check runs — a moved floor does not promote
an arm nobody used into something worth a line."""
ws = warn({"auto_inject": src(calls=1, zero_result_calls=0, p10=0.705)},
floors={"auto_inject": 0.70}, floor_moves={"auto_inject": MOVED})
assert "floor_moved_mid_window" not in codes(ws)