From 2b85443dd19f467666321529d83d41a5bd0112cb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 28 Jul 2026 18:09:17 -0400 Subject: [PATCH] =?UTF-8?q?feat(snippets):=20usage=20signal=20=E2=80=94=20?= =?UTF-8?q?was=20a=20surfaced=20record=20ever=20actually=20pulled=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retrieval_logs answers "what did the ranker return, at what scores" — the right substrate for tuning a threshold. It cannot answer the question the snippet corpus actually needs: did anyone open this? A snippet nobody opens is not neutral. It takes a slot in every future auto-inject menu and crowds out something useful. Adds note_usage_events (migration 0071): one row per note per event, either 'surfaced' (we put its title in front of an agent) or 'pulled' (someone opened it in full), tagged with which surface produced it. Closes the gap #2082 recorded against this work. The write-path PLACE arm carries no score, so it has no home in retrieval_logs — folding it in would corrupt the score distribution that table exists to capture. The result was that the arm firing on the STRONGEST claim ("there is already a canonical helper in this exact file") was the one arm nobody could measure. Both arms now emit usage events under distinct sources, so their pull-through rates are finally comparable. Deliberate departures from the task as written: - Not in-session correlation. The original framing was "correlate result_ids against a later get_note in the same session." There is no session identity server-side — the MCP endpoint is stateless and the hooks send no session id — and adding one would mean threading an opaque client-supplied token through every read path. Two independent counters answer the question without it: surfaced 40×, pulled 0 is dead weight regardless of how those events distribute across sessions. - Pulls record at the ENTRY POINTS (MCP tools, REST detail route), not in snippets_svc.get_snippet, which update and merge also reach. Counting those would inflate precisely the number meant to say "someone chose to look at this." - get_note records for every note kind, not just snippets. The auto-inject menu surfaces tasks and processes too; scoping this to snippets would pin those at zero pulls forever and make them read as dead weight next to snippets that merely had a counter. Surfaced in the Snippets list as an "N/M used" badge, warning-toned once a record has been offered 3+ times and never opened, with the tooltip saying what to do about it (usually: its "when to reach for it" doesn't say when). No badge at all below one surfacing — "0/0" reads as a verdict when it's an absence of evidence. Also returned from MCP list_snippets so the agent can see dead weight without opening the UI. Telemetry keeps the retrieval_telemetry contract throughout: writes are fire-and-forget, reads degrade to zeroes, and no path can raise into the surface it observes. Refs #2085 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs --- alembic/versions/0071_note_usage_events.py | 72 ++++++++ frontend/src/api/snippets.ts | 12 ++ frontend/src/views/SnippetListView.vue | 59 +++++++ src/scribe/mcp/tools/notes.py | 6 + src/scribe/mcp/tools/snippets.py | 27 ++- src/scribe/models/__init__.py | 1 + src/scribe/models/note_usage.py | 76 +++++++++ src/scribe/routes/snippets.py | 14 ++ src/scribe/services/note_usage.py | 155 +++++++++++++++++ src/scribe/services/plugin_context.py | 31 +++- tests/test_note_usage.py | 185 +++++++++++++++++++++ 11 files changed, 626 insertions(+), 12 deletions(-) create mode 100644 alembic/versions/0071_note_usage_events.py create mode 100644 src/scribe/models/note_usage.py create mode 100644 src/scribe/services/note_usage.py create mode 100644 tests/test_note_usage.py diff --git a/alembic/versions/0071_note_usage_events.py b/alembic/versions/0071_note_usage_events.py new file mode 100644 index 0000000..c9ac3cb --- /dev/null +++ b/alembic/versions/0071_note_usage_events.py @@ -0,0 +1,72 @@ +"""add note_usage_events — did anyone actually open what we surfaced? + +Revision ID: 0071 +Revises: 0070 +Create Date: 2026-07-28 + +`retrieval_logs` records what the ranker returned and with what scores, which is +the right substrate for tuning a similarity threshold. It cannot answer the +different question the snippet corpus needs: was a surfaced snippet ever pulled +in full? A snippet nobody opens still competes for the injection budget on every +turn, so the surfaced:pulled ratio is what makes dead weight visible. + +Two reasons this is its own table rather than columns on `notes` or rows in +`retrieval_logs`: + + - Counters on `notes` would answer "how many" but not "when, from where, and + by which arm" — and the place arm vs semantic arm comparison is precisely + what was missing (the write-path place arm surfaced snippets while leaving + no trace anywhere). + - Folding un-scored surfacing into `retrieval_logs` would corrupt the score + distribution that table exists to capture. Location hits have no score. + +Grain is one row per note per event, which is what the per-snippet readout needs +and what `retrieval_logs.result_ids` (a JSONB array, one row per *call*) cannot +be indexed at. + +FK-free on note_id and user_id, matching retrieval_logs and app_logs: telemetry +should outlive what it describes. Deleting a note must not erase the evidence +that it was surfaced forty times and opened none. + +Downgrade drops the table outright. The data is purely observational — nothing +reads it for correctness, so losing it costs history and no behavior. +""" +from alembic import op +import sqlalchemy as sa + + +revision = "0071" +down_revision = "0070" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "note_usage_events", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("note_id", sa.Integer(), nullable=False), + sa.Column("event", sa.Text(), nullable=False), + sa.Column("source", sa.Text(), nullable=False), + ) + # Every readout is "these note ids, split by event", so the composite is the + # one that actually gets used; the others serve pruning and per-user views. + op.create_index( + "ix_note_usage_note_event", "note_usage_events", ["note_id", "event"] + ) + op.create_index("ix_note_usage_created_at", "note_usage_events", ["created_at"]) + op.create_index("ix_note_usage_user_id", "note_usage_events", ["user_id"]) + + +def downgrade() -> None: + op.drop_index("ix_note_usage_user_id", table_name="note_usage_events") + op.drop_index("ix_note_usage_created_at", table_name="note_usage_events") + op.drop_index("ix_note_usage_note_event", table_name="note_usage_events") + op.drop_table("note_usage_events") diff --git a/frontend/src/api/snippets.ts b/frontend/src/api/snippets.ts index fa7ef03..e912d4c 100644 --- a/frontend/src/api/snippets.ts +++ b/frontend/src/api/snippets.ts @@ -46,6 +46,16 @@ export interface Snippet { owner?: string | null; } +/** How often a record was put in front of an agent versus actually opened. + * A high `surfaced_count` with `pull_count: 0` is dead weight — it occupies a + * slot in every future auto-inject menu while never being used. */ +export interface SnippetUsage { + surfaced_count: number; + pull_count: number; + last_surfaced_at: string | null; + last_pulled_at: string | null; +} + /** Lightweight list item from the knowledge preview feed. Note: the `snippet` * field here is a truncated *body preview* (the knowledge feed's naming), not * the parsed fields above. */ @@ -61,6 +71,8 @@ export interface SnippetListItem { * them, not one of your own. Absent means it's yours. */ shared?: boolean; owner?: string | null; + /** Always present from the backend, zero-filled for records with no events. */ + usage?: SnippetUsage; } /** Create/update payload — discrete fields the backend serializes into the diff --git a/frontend/src/views/SnippetListView.vue b/frontend/src/views/SnippetListView.vue index 8fbed78..2af502f 100644 --- a/frontend/src/views/SnippetListView.vue +++ b/frontend/src/views/SnippetListView.vue @@ -126,6 +126,40 @@ function splitTitle(title: string): { name: string; when: string } { function languageOf(tags: string[]): string { return tags.find((t) => t && t !== "snippet") ?? ""; } + +/** A snippet that has been offered repeatedly and never opened. The threshold + * is 3 rather than 1 because one or two surfacings is noise — the record may + * simply not have come up in a relevant context yet. */ +function isDeadWeight(s: SnippetListItem): boolean { + const u = s.usage; + return !!u && u.pull_count === 0 && u.surfaced_count >= 3; +} + +/** Short badge text, or "" to render nothing. A record nobody has surfaced yet + * gets no badge at all: "0 / 0" would read as a verdict when it's an absence + * of evidence. */ +function usageBadge(s: SnippetListItem): string { + const u = s.usage; + if (!u || u.surfaced_count === 0) return ""; + return `${u.pull_count}/${u.surfaced_count} used`; +} + +function usageTitle(s: SnippetListItem): string { + const u = s.usage; + if (!u) return ""; + const last = u.last_pulled_at + ? `Last opened ${new Date(u.last_pulled_at).toLocaleDateString()}.` + : "Never opened."; + const verdict = isDeadWeight(s) + ? " Offered repeatedly without ever being opened — consider rewriting its" + + " “when to reach for it” so it says when, or deleting it. It takes a slot" + + " in every future auto-inject menu." + : ""; + return ( + `Surfaced to an agent ${u.surfaced_count}×, opened in full ` + + `${u.pull_count}×. ${last}${verdict}` + ); +}