diff --git a/src/scribe/services/retrieval_telemetry.py b/src/scribe/services/retrieval_telemetry.py index 5762ae28..fdb45351 100644 --- a/src/scribe/services/retrieval_telemetry.py +++ b/src/scribe/services/retrieval_telemetry.py @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) # Strong references to in-flight inserts — the loop holds tasks only weakly, # and an unreferenced fire-and-forget task can be collected before it runs -# (same guard as note_usage, found via #2663). +# (same guard as note_usage). _pending: set[asyncio.Task] = set() # Whether this process already dropped its one warning about failing writes. @@ -61,15 +61,14 @@ _reported = False -# ── secrets never reach the query column (#3925) ─────────────────────── +# ── secrets never reach the query column ─────────────────────────────── # # `pre_tool_rule` retrieves against the RAW COMMAND TEXT and `write_path_rule` # against the code being written, so whatever was on the command line or in the -# buffer is what gets logged. A command that exports a token therefore stored -# the token — and worse than stored it: `near_miss_samples` is the readout the -# threshold docs tell you to open before moving a bar, so the value came back -# out into an agent's context on the next tuning pass. That is how this was -# found. +# buffer is what gets logged. Unscrubbed, a command that exports a token would +# store the token — and worse than store it: `near_miss_samples` is the +# readout the threshold docs tell you to open before moving a bar, so the +# value would come back out into an agent's context on the next tuning pass. # # SCRUBBED ON WRITE, NOT ON READ. A read-side filter leaves the secret in the # table, where a backup, a debug query or a future readout still reaches it. @@ -78,8 +77,8 @@ _reported = False # REDACTED VISIBLY, AND THIS IS THE PART THAT KEEPS THE READOUT HONEST. The # whole worth of a near-miss sample is reading the query that was actually # refused; a scrubber that silently deleted spans would turn the one instrument -# for tuning a bar into unreadable stubs — the #2663 shape, where a surface -# looks fine and has quietly stopped saying anything. A `[redacted:]` +# for tuning a bar into unreadable stubs — a surface that looks fine and has +# quietly stopped saying anything. A `[redacted:]` # marker keeps the sentence readable, keeps its shape and length roughly # intact for the ranker's reader, and says plainly that something was removed. # @@ -194,7 +193,7 @@ def _build_payload( "best_available_score": ( None if best_available is None else round(float(best_available), 5) ), - # The record that scored it, so a reader can go and look (#3807). + # The record that scored it, so a reader can go and look. "best_available_id": ( None if best_available_id is None else int(best_available_id) ), @@ -208,7 +207,7 @@ async def _insert_retrieval_log(payload: dict) -> None: WARNING rather than debug — this table is the empirical basis for threshold tuning, and a silent write outage yields a dataset that looks complete while - covering only part of the traffic (#2663's shape). Once per process is + covering only part of the traffic. Once per process is enough to be found; per-call would flood the log with what it already said. """ global _reported @@ -243,7 +242,7 @@ def record_retrieval( """Fire-and-forget: record one retrieval call. `results` needs only `.id` on each record, which is why it is not typed to - Note: rules are retrieved too (milestone 307) and land here rather than in + Note: rules are retrieved too and land here rather than in note_usage_events. That table's ids are REMAPPED on a backup restore, so a rule id written into it would come back attached to whatever note happened to take that number — silent corruption of the very evidence this exists to @@ -253,13 +252,12 @@ def record_retrieval( `searched=False` WRITES NO ROW, and that is the point rather than an optimisation. A semantic search has three ways to return nothing without having run — an empty query, an unavailable embedder, and the broad - `except` around the query itself — and each one currently arrives here - looking exactly like a ranker that declined. Logging it would report a - decline nobody made, drag `zero_result_calls` down with phantom evidence - about a threshold, and leave `best_available_score` null for a reason that - has nothing to do with the corpus. That last ambiguity is #3765: the field - added to judge a bar was null on four unrelated causes, one of them a - swallowed failure, and no reader could tell them apart. + `except` around the query itself — and each one arrives here looking + exactly like a ranker that declined. Logging it would report a decline + nobody made, drag `zero_result_calls` down with phantom evidence about a + threshold, and leave `best_available_score` null for a reason that has + nothing to do with the corpus — one of four unrelated causes, one of them + a swallowed failure, that no reader could tell apart. Dropping the row is what makes the remaining nulls mean ONE thing — "searched, and there was nothing". @@ -309,13 +307,11 @@ def record_retrieval( task.add_done_callback(_pending.discard) -# --- The read half (#2975) --------------------------------------------------- -# Until this existed, `retrieval_logs` was WRITE-ONLY: rows accrued and the only -# `select()` over them in the whole tree lived in a test. That made #1038's gate -# — "build the reranker once telemetry shows precision is the bottleneck" — -# unsatisfiable by construction, and it is why the one real tuning decision on -# record (the 0.68 write-path threshold, #2223) was reached by hand-probing the -# live instance with eight payloads instead of by reading what was collected. +# --- The read half ----------------------------------------------------------- +# What makes `retrieval_logs` evidence rather than a write-only table: a +# threshold is tuned by reading what was collected, not by hand-probing an +# instance with a few payloads, and "build a reranker once precision is the +# bottleneck" needs a readout that can show precision is the bottleneck. def _bucket(rows: list) -> dict: """A score readout a human can act on, from one aggregate row.""" @@ -328,16 +324,13 @@ def _bucket(rows: list) -> dict: # different failure (nothing indexed, filter too narrow), and averaging # it into the score distribution would hide both. "zero_result_calls": int(zero or 0), - # `cleared_threshold` USED TO LIVE HERE and it was a tautology (#3670). - # The search applies the bar before returning, so every returned result + # NO "cleared the threshold" COUNT: it would be a tautology. The + # search applies the bar before returning, so every returned result # cleared it by construction and a call with nothing has no score to - # compare — the condition was true exactly when `result_count > 0`. - # `zero_result_calls + cleared_threshold == calls` held on all nineteen - # readings ever taken. It was `calls - zero_result_calls` wearing a name - # that promised a second opinion, and the docstring built a reading - # procedure on it that asked the reader to compare a number with itself. - # Its replacement is `near_misses` below, which the bar cannot fix by - # construction because it is measured on the calls the bar REJECTED. + # compare — the count would be `calls - zero_result_calls` wearing a + # name that promises a second opinion. `near_misses` below is the + # measure the bar cannot fix by construction, because it is taken on + # the calls the bar REJECTED. # Of the zeros above, which were the RANKER declining and which were # the reader having seen it already? `zero_result_calls` cannot say, # and only the first kind is evidence about the threshold. @@ -345,7 +338,7 @@ def _bucket(rows: list) -> dict: # None — not a zeroed dict — when no row in the window reported it. A # surface that filters inside the search genuinely does not know, and # rendering that as `{"calls": 0}` would state a measurement nobody - # made. That substitution is the whole of #3311. + # made. "suppression": ( None if not int(measured or 0) else { "measured_calls": int(measured or 0), @@ -438,11 +431,11 @@ def _coverage(complete_from, since) -> dict: `covers_window` is None — never False — when nothing was ever recorded. "No rows at all" is not "partial coverage", it is no measurement, and the - null convention #3497 established for `suppression` holds here for the + null convention `suppression` follows holds here for the same reason: absent must not read as a verdict. """ return { - # iso() already returns None for an unset value (#2845) — the guard + # iso() already returns None for an unset value — the guard # belongs on covers_window, which is a verdict, not a serialisation. "complete_from": iso(complete_from), "covers_window": ( @@ -451,20 +444,19 @@ def _coverage(complete_from, since) -> dict: } -# ── What is wrong, computed rather than re-derived by hand (#3431) ──────── +# ── What is wrong, computed rather than re-derived by hand ──────────────── # -# Every reading of this tool used to be a hand analysis — #3430's baseline, -# #3835's rule near-misses, the #1038 rerank gate — and the analysis was the -# same four checks each time. Worse than tedious: it was UNRELIABLE, because -# the reader had to remember them. The checks are mechanical, so they belong -# in the tool, phrased for an agent reading the output rather than a page. +# Reading this tool by hand means running the same four checks each time, and +# a check the reader has to remember is UNRELIABLE. The checks are +# mechanical, so they belong in the tool, phrased for an agent reading the +# output rather than a page. # HOW MUCH TRAFFIC BEFORE A SILENCE MEANS ANYTHING. Three calls returning # nothing is a quiet afternoon; three hundred is an arm that has lost its # voice, and only the count separates them. Settings-backed because the right -# number depends on how hard an install is driven (rule 25), and defaulted -# high enough that a FRESH INSTALL WITH ALMOST NO DATA PRODUCES NO WARNINGS AT -# ALL (rule 115) — a new user's first readout saying five things are broken +# number depends on how hard an install is driven, and defaulted high enough +# that a FRESH INSTALL WITH ALMOST NO DATA PRODUCES NO WARNINGS AT ALL — a new +# user's first readout saying five things are broken # would be describing the emptiness, not the system. WARN_MIN_CALLS_KEY = "retrieval_warn_min_calls" WARN_MIN_CALLS_DEFAULT = 30 @@ -473,8 +465,8 @@ WARN_MIN_CALLS_DEFAULT = 30 # arm returns still sits within a hair of the floor, the score is not sorting # anything — everything it admits is borderline, and the bar is doing the # whole job the ranking was supposed to do. 0.02 is roughly the spread this -# corpus shows between a genuine match and an incidental one (#2485 measured -# the top-to-second gap at 0.010-0.023 for everything but snippets), so a +# corpus shows between a genuine match and an incidental one (the +# top-to-second gap measures 0.010-0.023 for everything but snippets), so a # band tighter than that is indistinguishable from noise. WARN_FLOOR_EPSILON_KEY = "retrieval_warn_floor_epsilon" WARN_FLOOR_EPSILON_DEFAULT = 0.02 @@ -485,7 +477,7 @@ def _num(raw: str, fallback): A malformed setting must not take the readout down with it — same posture as the rest of this module, where a telemetry failure that breaks its - caller is the worse bug (#2663). + caller is the worse bug. """ try: return type(fallback)(raw) @@ -495,11 +487,11 @@ def _num(raw: str, fallback): # What "surfaced and never opened" can and cannot mean, per corpus, and where # to look next. ONE SENTENCE PER CORPUS because the two lines are built -# differently (#4798): a note line carries its matched passage, so an unopened +# differently: a note line carries its matched passage, so an unopened # note may have done its job, and the judged sample is the instrument that can # tell. A rule line carries only its trigger and asks to be opened, and no # judged sample covers the rule arms (`retrieval_review.REVIEWABLE`), so -# sending that reader to `menus_to_review` sent them to a refusal. +# sending that reader to `menus_to_review` would send them to a refusal. _NEVER_PULLED_READING = { "notes": ( "That is not a verdict on them: a line carries its matched passage, " @@ -581,15 +573,14 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, # it is stuck — either its bar is beneath everything or it is not # applying one. # - # THREE GUARDS BEFORE TRUSTING THIS, and each is a bug it already - # caused. Asked surfaces are exempt: a search returning a list every + # THREE GUARDS BEFORE TRUSTING THIS, and each prevents a false + # alarm. Asked surfaces are exempt: a search returning a list every # time is a search doing its job, and flagging `mcp_search` would # teach the reader to skip the list. An UNREGISTERED source is exempt # because nothing says which kind it is, and guessing from the name is - # the narrowing #3191 warns about. And an arm not known to log - # unconditionally is exempt because that was #3497 exactly: both rule - # arms once recorded only their hits, so their decline count was - # structurally zero and this warning would have fired on a LOGGING + # a silent narrowing. And an arm not known to log unconditionally is + # exempt: an arm that records only its hits has a decline count that + # is structurally zero, and this warning would fire on a LOGGING # defect while pointing the reader at the threshold. if ( calls >= min_calls @@ -623,16 +614,15 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, if calls >= min_calls and floor is not None and p10 is not None: # ── The floor moved inside the window ──────────────────────── # - # THE CHECK IS SUSPENDED, NOT SOFTENED, and this is the whole of - # #4225. `band_hugs_floor` asks whether the scores are piled on + # THE CHECK IS SUSPENDED, NOT SOFTENED. `band_hugs_floor` asks + # whether the scores are piled on # the bar. That question needs the scores and the bar to come from # the same regime; across a floor change they do not, and the # comparison quietly becomes one between two populations. # - # It announced itself when the change was a RAISE: p10 computed - # over calls made under the old, lower bar came out BELOW today's - # floor, and the warning reported a band "-0.0216 above" its - # floor. A negative distance above something is not a number + # After a RAISE it shows: p10 computed over calls made under the + # old, lower bar comes out BELOW today's floor, and the warning + # would report a band "-0.0216 above" its floor — not a number # anybody can act on. A LOWERED floor hides better — the gap comes # out comfortably positive and reads as a clean bill of health on # a sample that half predates the bar being judged. @@ -725,11 +715,9 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, # ── Surfaced and never pulled, for each corpus ─────────────────────── # # The one corpus-level check, and the only number here that judges the - # RECORDS rather than the arms. It used to say a record shown repeatedly - # and never opened is "badly titled or genuinely irrelevant" — true when a - # line was a title, and not since it carries the passage (#4364): the - # record may have done its job unopened. So it now points at the judged - # sample rather than at a cause (#4772). + # RECORDS rather than the arms. A line carries the matched passage, so a + # record shown repeatedly and never opened may have done its job unopened; + # the warning points at the judged sample rather than at a cause. # # Distinct records, not events: a note surfaced forty times and never # opened is one problem, not forty. @@ -751,13 +739,12 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, surfaced=int(shown), pulled=int(pulled or 0), never_pulled=never, )) - # ── A rule that was read and changed nothing (#4213, milestone 419) ── + # ── A rule that was read and changed nothing ───────────────────────── # - # The failure this milestone was opened on, and the one number that could - # not previously be computed. `surfaced_never_pulled` above catches a rule - # nobody opens; this catches the worse case — a rule the agent DID open, - # deliberately, and then left no trace of having acted on. Until #4212 - # those were arithmetically identical to compliance. + # `surfaced_never_pulled` above catches a rule nobody opens; this catches + # the worse case — a rule the agent DID open, deliberately, and then left + # no trace of having acted on. Without recorded outcomes those are + # arithmetically identical to compliance. opened = int(rule_usage.get("distinct_rules_pulled") or 0) acted = int(rule_usage.get("distinct_rules_acted") or 0) applied = int(rule_usage.get("applied") or 0) @@ -766,12 +753,10 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict, if opened and not (applied or departed): # THE HONEST ANSWER WHILE THE INSTRUMENT IS COLD, and the reason this # is a separate code rather than a zero fed into the check below. - # Outcomes only started being recorded in milestone 419; a window - # containing none cannot tell "every rule was ignored" from "nothing - # reports outcomes yet". Emitting the ignored-rules warning here would - # manufacture a finding out of an unwired feature — #3311's mistake - # exactly, where a statistic that could not vary was read as a fact - # about the corpus. + # A window containing no outcomes cannot tell "every rule was + # ignored" from "nothing reports outcomes". Emitting the ignored-rules + # warning here would manufacture a finding out of an unwired feature — + # a statistic that cannot vary, read as a fact about the corpus. out.append(_warn( "outcomes_never_recorded", f"{opened} distinct rules were opened in this window and not one " @@ -806,12 +791,12 @@ def _silent_surfaces(sources: dict, usage: dict, rule_usage: dict, THE HALF THE ROWS CANNOT SEE. Every check above reads rows, so an arm that produced none is invisible to all of them — it looks identical to an arm - that does not exist. #3430 found one of these, and only because a human - happened to know the arm was supposed to be there. + that does not exist, and is found otherwise only by a human who happens + to know the arm is supposed to be there. ONLY WHEN THE INSTALL IS OTHERWISE ACTIVE. On a quiet window every point - is silent and the list would be the registry, printed back. Rule 115: a - fresh install must not be told that thirty things are broken when the + is silent and the list would be the registry, printed back. A fresh + install must not be told that thirty things are broken when the truth is that nobody has used it yet. """ if not active: @@ -832,12 +817,12 @@ SYSTEM_USAGE_TOP = 20 async def _system_usage(user_id: int | None, since) -> dict: - """The rulings arm's own block (#4769): were an area's rulings shown, and + """The rulings arm's own block: were an area's rulings shown, and was the area then opened. FROM `system_usage_events`, IN ITS OWN SESSION AND ITS OWN GUARD, for the reason the rule block is guarded apart: a failure here must not take down - the readouts beside it, and must say so rather than print zeros (#2663) — + the readouts beside it, and must say so rather than print zeros — the keys are kept and `system_usage_failed` is added. NO PULL-THROUGH, and the omission is the design. The note and rule ratios @@ -938,7 +923,7 @@ async def _system_usage(user_id: int | None, since) -> dict: async def moment_usage(user_id: int | None, since) -> dict: - """Deliveries and opens per moment (milestone 458 step 6). + """Deliveries and opens per moment. A mounted rule's delivery is a `moment_rule` surfacing whose `detail` is the moment it arrived at — the plugin's catch-all hook, the MCP tools' @@ -960,7 +945,7 @@ async def moment_usage(user_id: int | None, since) -> dict: question about the rule's wording or the agent, not a bar to tune. The counts sit side by side. - Its own session and guard (#2663): a failure keeps the shape and adds + Its own session and guard: a failure keeps the shape and adds `moment_usage_failed`, never zeros that read as "nothing happened". """ from scribe.services.retrieval_pipeline import MOMENT_RULE_SOURCE @@ -1032,7 +1017,7 @@ async def retrieval_summary( Three aggregates side by side, each read from the table built for it — NOT a join. `usage` is notes, `rule_usage` is rules, and they stay apart because a few dozen eligible rules blended into thousands of notes is the - note ratio with noise on it (milestone 333). `NoteUsageEvent`'s own docstring is explicit that the two are + note ratio with noise on it. `NoteUsageEvent`'s own docstring is explicit that the two are complements ("RetrievalLog tunes the threshold, this tunes the corpus") and that RetrievalLog's JSONB `result_ids` "can't be indexed at" the per-note grain. So the score distribution comes from `retrieval_logs` on its indexed @@ -1049,11 +1034,11 @@ async def retrieval_summary( Scoped to one user's own telemetry. There is no sharing model for a retrieval log — it records what THIS user's agent asked for, including the query text — so an owner filter is the whole access rule here rather than a - shortcut around `services/access.py` (P#78 governs shared record kinds). + shortcut around `services/access.py` (which governs shared record kinds). Never raises: a telemetry readout that can break its caller is worse than no readout. It does distinguish "no rows" from "the read failed", because - #2663 is exactly the bug where those two looked identical for weeks. + a read outage that looks like a quiet window can go unnoticed for weeks. """ since = datetime.now(timezone.utc) - timedelta(days=max(1, int(days))) out: dict = { @@ -1069,24 +1054,21 @@ async def retrieval_summary( zero = case((RetrievalLog.result_count == 0, 1), else_=0) # THE NEAR-MISS POPULATION: calls that returned nothing BECAUSE THE BAR - # TURNED SOMETHING AWAY, and recorded what it was. Three conditions, and - # the third was missing for one deploy (#3739). + # TURNED SOMETHING AWAY, and recorded what it was. Three conditions. # # Zero-result only: on a call that returned something, # `best_available_score` equals `top_score` and adds nothing. # - # Non-null only: rows written before #3670 genuinely do not know, and must - # not read as scoreless declines. + # Non-null only: rows written before the score was recorded genuinely do + # not know, and must not read as scoreless declines. # # AND NOT A REPEAT. A zero-result call is two unrelated events — the ranker # found nothing above the bar, or it found only what this session had - # already been shown — and just the first says anything about the bar. That - # is the whole of #3497, and #3670 reintroduced the conflation one level up: - # the rule arms filter exclusions in PYTHON, after the search, so a rule - # that cleared the bar and was dropped as a repeat still reported a high - # `best_available_score` on a zero-result row. Live proof, first read after - # deploy: pre_tool_rule's near-miss max was 0.7457 while the lowest score it - # ever RETURNED was 0.7204 — a "rejection" that outscored acceptances. + # already been shown — and just the first says anything about the bar. + # The rule arms filter exclusions in PYTHON, after the search, so a rule + # that cleared the bar and was dropped as a repeat still reports a high + # `best_available_score` on a zero-result row — a "rejection" that + # outscores acceptances. # # The NULL arm is principled, not permissive: `suppressed_count IS NULL` # means the caller passed its exclusions INTO the search, which is exactly @@ -1126,7 +1108,7 @@ async def retrieval_summary( return func.percentile_cont(p).within_group(RetrievalLog.top_score.asc()) # Assigned inside the try below; named here so the readout can tell - # "this query failed" from "this window has no rows" (#2663). + # "this query failed" from "this window has no rows". by_source_rows = None rule_rows = None distinct_rules_surfaced = distinct_rules_pulled = 0 @@ -1177,8 +1159,8 @@ async def retrieval_summary( # A source with rows in the table but NONE in this window would # otherwise be absent from the readout — and absent is exactly how # a source that never existed renders, so a surface that WAS - # recording and went silent is unreadable (#3720). That is #2663 - # one level up: the failure that looks like the correct answer. + # recording and went silent is unreadable — the failure that looks + # like the correct answer. # # Zero here is a real measurement, not a manufactured one. The # all-time query proves the source was recording, and it made no @@ -1193,18 +1175,17 @@ async def retrieval_summary( quiet.update(_coverage(first_row, since)) out["sources"][src] = quiet - # WHAT THE BAR REFUSED, by name (#3807). Opt-in, because it is a + # WHAT THE BAR REFUSED, by name. Opt-in, because it is a # LISTING and not a statistic: an id cannot be percentiled, and a # reader tuning a threshold needs to go and read the records rather # than see another number about them. Off by default so the # ordinary readout keeps its size. # - # Deliberately NOT a window function. This module's one production - # outage (#2663) was a grouped query the database rejected, swallowed - # by the broad except, every counter reading zero while the mocked - # tests passed — and the lesson recorded then was to group on a raw - # column and classify in Python rather than push cleverness into the - # SQL. So: one flat ordered query, overfetched, bucketed here. + # Deliberately NOT a window function. A clever query the database + # rejects is swallowed by the broad except, and every counter reads + # zero while mocked tests pass; grouping on a raw column and + # classifying in Python cannot fail that way. So: one flat ordered + # query, overfetched, bucketed here. if near_miss_samples > 0: want = max(1, min(int(near_miss_samples), 20)) rows = ( @@ -1245,11 +1226,11 @@ async def retrieval_summary( # note_usage.usage_for_notes: an ambient surfacing was not a scored # CHOICE, so folding it into pull-through would understate it. # Grouped by RAW source, then classified in Python. The - # alternative — CASE expressions in the GROUP BY — is the shape - # that produced #2663: a second case() renders its own expanding - # bind names, the database sees two different expressions and - # rejects the query, and the broad except swallows it. One CASE is - # provably fine (usage_for_notes does it); two is where it broke. + # alternative — CASE expressions in the GROUP BY — fails silently: + # a second case() renders its own expanding bind names, the + # database sees two different expressions and rejects the query, + # and the broad except swallows it. One CASE is fine + # (usage_for_notes does it); two is where it breaks. # `source` has a handful of distinct values, so grouping on it # directly is cheap and cannot fail that way at all. urows = ( @@ -1295,27 +1276,26 @@ async def retrieval_summary( ) ).scalar_one() - # Per-source pull-through, at the NOTE grain (#3311). + # Per-source pull-through, at the NOTE grain. # - # The `urows` query above already groups by source and the loop - # below then throws the source away, so until now this readout - # could say what the corpus's overall pull-through was and nothing - # about WHICH surface earned it. The data was always here; only - # the aggregation discarded it. + # The `urows` query above groups by source and the loop below + # throws the source away, so without this the readout could say + # what the corpus's overall pull-through was and nothing about + # WHICH surface earned it. # # It cannot be had by grouping the PULLED rows by source: a pull # records the door it came through (`mcp_get_note`), not the # surface that put the record in front of the agent. Correlating - # those within a session is what #2085 ruled out — there is no + # those within a session is ruled out — there is no # session identity server-side and inventing one would mean # threading a client-supplied token through every read path. The # note grain answers the question without one: of the distinct # notes surface X chose, how many did an agent open in this window? # - # Guarded separately from the reads above, on #2663's actual - # lesson. That outage was a NOVEL SQL SHAPE the database rejected - # inside a broad except. This join is the novel shape here, and a - # failure in it must not take down two readouts that already work. + # Guarded separately from the reads above. A NOVEL SQL SHAPE the + # database rejects inside a broad except zeroes everything in that + # except. This join is the novel shape here, and a failure in it + # must not take down two readouts that already work. try: pulled_ids = ( select(NoteUsageEvent.note_id) @@ -1366,12 +1346,12 @@ async def retrieval_summary( logger.warning("per-source pull-through read failed", exc_info=True) by_source_rows = None - # Rules, at their own grain and in their own block (milestone 333). + # Rules, at their own grain and in their own block. # # Guarded separately from the reads above for the reason `by_source` - # is: this table is NEW, and an instance running upgraded code - # against un-migrated schema would otherwise take down two readouts - # that work perfectly in order to report a third that cannot. + # is: an instance running upgraded code against un-migrated schema + # would otherwise take down two readouts that work in order to + # report a third that cannot. # # The queries themselves are the note block's shapes, not novel # ones — a group-by on two indexed columns and two count(distinct). @@ -1398,8 +1378,8 @@ async def retrieval_summary( session, RuleUsageEvent, user_id, ) # The rows carry `source`, so the ranked/ambient split is done - # below rather than in SQL — the bulk surfaces started emitting - # on 2026-09-03 (#3473), so there IS an ambient class now. + # below rather than in SQL — the bulk surfaces emit ambient + # rows beside the ranked ones. # # `distinct_rules_surfaced` deliberately counts BOTH classes. It # answers "how many distinct rules did this install put in front @@ -1426,7 +1406,7 @@ async def retrieval_summary( ) ) ).scalar_one() - # Rules that were OPENED AND THEN ACTED ON (#4212). Its own + # Rules that were OPENED AND THEN ACTED ON. Its own # distinct count for the same reason the two above have one: # "how many rules did anything come of" cannot be summed from # the per-source group without double-counting a rule that was @@ -1469,7 +1449,7 @@ async def retrieval_summary( elif event == PULLED: usage["pulled"] += n # The mcp_/rest_ split is load-bearing (see NoteUsageEvent's own - # comment, which names #1038 — this readout's whole purpose). "Is + # comment). "Is # this record dead weight?" is answered by ANY pull; "was that # injected line useful to the agent?" only by an AGENT pull. So # pull-through, which exists to answer the second, counts mcp_* @@ -1494,8 +1474,8 @@ async def retrieval_summary( # # UPPER BOUND, and say so where it will be read: a pull records the door, # not the surface that led to it, so a note surfaced by two surfaces and - # opened once counts as pulled for both. Attribution would need the session - # identity #2085 declined to invent. The bound is still decisive in the + # opened once counts as pulled for both. Attribution would need a session + # identity the server does not have. The bound is still decisive in the # direction that matters — a surface reading near zero here is not being # flattered by the double-count. if by_source_rows is None: @@ -1539,12 +1519,9 @@ async def retrieval_summary( # been comparing across windows, without telling them it now measures # something else. # - # `ambient` now carries the bulk deliveries — the SessionStart preload, - # and every `rules_payload` surface (#3473). Before - # they emitted, this block had no ambient key and said the absence was a - # fact about the data. It was, and it was also the thing that made the - # always-on set impossible to judge: the largest rule surface in the - # product was the one surface its own scoreboard could not see. + # `ambient` carries the bulk deliveries — the SessionStart preload, and + # every `rules_payload` surface. Without it the largest rule surface in + # the product would be the one its own scoreboard could not see. # # READ THE TWO SEPARATELY, ALWAYS. `surfaced` is a claim a ranker made and # a pull can settle. `ambient` is a delivery nobody chose, so a high count @@ -1554,10 +1531,10 @@ async def retrieval_summary( "pulled": 0, "pulled_by_agent": 0, "pulled_by_human": 0, "distinct_rules_surfaced": int(distinct_rules_surfaced or 0), "distinct_rules_pulled": int(distinct_rules_pulled or 0), - # The outcome half (#4212, milestone 419). `pulled` says a rule was - # opened; these say whether anything came of it. Until this existed, a - # rule obeyed every time and a rule ignored every time produced - # identical rows, and the second is the one worth finding. + # The outcome half. `pulled` says a rule was opened; these say + # whether anything came of it. Without them a rule obeyed every time + # and a rule ignored every time produce identical rows, and the second + # is the one worth finding. "applied": 0, "departed": 0, "distinct_rules_acted": int(distinct_rules_acted or 0), @@ -1567,7 +1544,7 @@ async def retrieval_summary( # one block up. A caller that renders this must not have to choose # between crashing on a missing key and quietly showing zeros it has no # right to: the keys let it render, and the flag tells it the zeros are - # "we could not find out" rather than "nothing happened" (#2663). + # "we could not find out" rather than "nothing happened". rule_usage["rule_usage_failed"] = True else: for event, source, n in rule_rows: @@ -1575,8 +1552,8 @@ async def retrieval_summary( if event == RULE_SURFACED: # One definition of ranked-vs-ambient, imported rather than # restated — the per-rule badge readout reads the same - # predicate, and two spellings of "what counts as surfaced" is - # precisely the uneven wiring #3246 found across this system. + # predicate, and two spellings of "what counts as surfaced" + # drift into readouts that disagree. if is_ambient(source): rule_usage["ambient"] += n else: @@ -1624,19 +1601,19 @@ async def retrieval_summary( out["system_usage"] = await _system_usage(user_id, since) out["moment_usage"] = await moment_usage(user_id, since) - # ── Judged lines (#4772) ───────────────────────────────────────────── + # ── Judged lines ───────────────────────────────────────────────────── # # The relevance readout the usage blocks cannot be: a reviewer's verdict # on each line of a sample of logged menus, by rank. Its own guard, for # `_system_usage`'s reason — a failure flags itself rather than printing - # an empty block, which reads as "nothing judged" (#2663). + # an empty block, which reads as "nothing judged". try: out["judged"] = await judged_block(user_id, since) except Exception: logger.warning("judged block read failed", exc_info=True) out["judged"] = {"judged_failed": True} - # ── What is wrong (#3431) ──────────────────────────────────────────── + # ── What is wrong ──────────────────────────────────────────────────── # # Computed LAST, over the blocks above rather than over the database, so # a warning can never disagree with the numbers printed beside it. Read @@ -1651,7 +1628,7 @@ async def retrieval_summary( # A FAILED READ JUDGES NOTHING. Warnings computed over rows that could not # be loaded would read as findings about the system rather than about the - # outage, which is the #2663 confusion arriving one level up. + # outage — an outage mistaken for a finding. if out["read_failed"]: return out @@ -1678,11 +1655,11 @@ async def retrieval_summary( except Exception: # pragma: no cover - telemetry never raises logger.warning("could not read floor for %s", name, exc_info=True) - # Floors that MOVED inside this window (#4225) — a release's shipped - # default or the operator's own dial, because the question is about the - # sample, not about who is answerable for it. Read unconditionally rather - # than under `if user_id`, unlike `floors` above: a release change belongs - # to no account, and it is exactly the case that used to go unrecorded. + # Floors that MOVED inside this window — a release's shipped default or + # the operator's own dial, because the question is about the sample, not + # about who is answerable for it. Read unconditionally rather than under + # `if user_id`, unlike `floors` above: a release change belongs to no + # account. floor_moves: dict[str, str] = {} try: floor_moves = await floor_moves_since(since) @@ -1692,7 +1669,7 @@ async def retrieval_summary( # And the arms whose floor history does not REACH the start of the window. # `floor_moves` answers "did it move"; for a window that opens before this # arm's first ledger row, that answer is unavailable rather than no — and - # the band check was reading the two as the same. Read the same way and for + # the band check must not read the two as the same. Read the same way and for # the same reason: the ledger belongs to the install, not to an account. floor_gaps: dict[str, str | None] = {} try: @@ -1707,7 +1684,7 @@ async def retrieval_summary( # "Active" means this window saw real traffic SOMEWHERE. Without that # test, an install nobody has used reports every registered point as - # silent — thirty warnings describing an empty database (rule 115). + # silent — thirty warnings describing an empty database. active = ( sum((b.get("calls") or 0) for b in out["sources"].values()) >= min_calls or (usage.get("surfaced") or 0) > 0