diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 07268f1..f6bc7e8 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "scribe", "description": "Scribe for Claude Code: connects the scribe MCP server, adds the hooks that deliver live project state and relevant records at the right moment, ships the shared client-neutral Scribe skills (using-scribe, writing-plans, reporting-back, systematic-debugging, verification, brainstorming, reusing-code, shape-accounting), and syncs your saved Scribe Processes as skills (/scribe:sync).", - "version": "2026.09.20.1623", + "version": "2026.09.20.2244", "author": { "name": "Bryan Van Deusen" }, diff --git a/plugin/hooks/scribe_autoinject.sh b/plugin/hooks/scribe_autoinject.sh index e960efe..48193a9 100755 --- a/plugin/hooks/scribe_autoinject.sh +++ b/plugin/hooks/scribe_autoinject.sh @@ -53,6 +53,14 @@ event_cwd=$(scribe_json_pick "$event_flat" '.cwd') # Nothing to retrieve against. [ -n "$prompt" ] || exit 0 +# A turn CLAUDE CODE wrote, not the operator (#4200) — a task notification, a +# slash-command echo, a local command's caveat banner. Leaving before the +# request matters more for the LOG than for the noise: a retrieval call +# recorded against a notification inflates this surface's denominator and +# seeds `near_misses` with demand nobody expressed. scribe_skip_prompt carries +# the measurement. +scribe_skip_prompt "$prompt" && exit 0 + # Unconfigured install → silent (auto-inject is pure enrichment). scribe_config || exit 0 diff --git a/plugin/hooks/scribe_defs.sh b/plugin/hooks/scribe_defs.sh index 33b950f..7d1b99d 100644 --- a/plugin/hooks/scribe_defs.sh +++ b/plugin/hooks/scribe_defs.sh @@ -8,6 +8,8 @@ # same three things, kept here so they cannot drift apart: # # scribe_skip_path PATH formats that hold prose or data, not shapes +# scribe_skip_prompt TEXT the turn was written by the CLIENT, not the +# operator — skip it entirely (#4200) # scribe_defs stdin code → "kindname" per definition # scribe_local_dups ROOT REL "kindname" lines on stdin → the by-name # local-duplicate lines (ARM 1, #2280) @@ -43,6 +45,47 @@ scribe_skip_path() { return 1 } +# Skip a turn the CLIENT wrote rather than the operator (#4200). Claude Code +# submits several kinds of machine-written text through UserPromptSubmit — a +# task notification, the echo of a slash command, the caveat banner a local +# command prints — and they reach `.prompt` indistinguishable from typed words +# unless something looks. +# +# THIS IS A TELEMETRY FIX FIRST AND A NOISE FIX SECOND, which is the part +# worth keeping straight: retrieving against a notification usually returns +# nothing, so the visible cost looks like one wasted embedding. The real cost +# is that the call is LOGGED. It inflates the denominator of every +# prompt-boundary surface, so delivery rate reads low for a reason that has +# nothing to do with retrieval; and the refusal lands in `near_misses`, where +# a later tuning decision reads it as unmet demand. Measured on #3898: 15 of +# the top 20 preference near-misses were `` blocks, ALL +# matching one record, ALL within thousandths of the floor. A floor lowered to +# serve that apparent demand would inject that record into every notification +# — the instrument arguing for the wrong fix, which is #379's lesson again. +# +# A PREFIX TEST, NOT A SUBSTRING ONE, and that distinction is the whole safety +# argument. A real prompt may well CONTAIN one of these tags — an operator +# pasting a transcript, or a `` appended after typed words — +# and must still be retrieved against. Nothing an operator types BEGINS with a +# client envelope. +# +# Leading whitespace goes first: the tag is what identifies the turn, and one +# stray newline ahead of it would otherwise walk past the entire filter. +# +# NOT LISTED, ON PURPOSE: the compaction-resume injection. It is machine +# written too, but it is a summary of real work rather than plumbing, and a +# resumed session is exactly where recalling a rule earns its keep. +scribe_skip_prompt() { + _scribe_prompt=${1#"${1%%[![:space:]]*}"} + case "$_scribe_prompt" in + ""*|\ + ""*|""*|""*|\ + ""*|""*|""*) + return 0 ;; + esac + return 1 +} + # --------------------------------------------------------------------------- # JSON AND URL-ENCODING, WITHOUT jq (#4107). # diff --git a/tests/test_prompt_boundary_filter.py b/tests/test_prompt_boundary_filter.py new file mode 100644 index 0000000..bd44a40 --- /dev/null +++ b/tests/test_prompt_boundary_filter.py @@ -0,0 +1,117 @@ +"""The prompt boundary retrieves against PROMPTS, not against plumbing (#4200). + +Claude Code submits more than typed words through `UserPromptSubmit`. A task +notification, the echo of a slash command and the caveat banner a local command +prints all arrive as user turns, and all reach `.prompt` looking exactly like +something the operator wrote. `scribe_autoinject.sh` believed them. + +WHY THIS IS TESTED AT ALL, given that retrieving against a notification almost +never returns anything. The cost that matters is not the wasted embedding, it +is the LOG ROW. Every one of those calls counts in the denominator of every +prompt-boundary surface, so delivery rate reads low for a reason unrelated to +retrieval quality; and each refusal lands in `near_misses`, where a later +tuning decision reads it as demand. Measured on #3898: 15 of the top 20 +preference near-misses were `` blocks, all matching one +record, all within thousandths of the floor. A floor lowered to serve that +apparent demand would have injected that record into every notification — an +instrument arguing for the wrong fix, which is exactly #379. + +So the assertions split in two, and the SECOND half is the one guarding +against the obvious wrong implementation: + + * the client's envelopes are skipped; + * a real prompt that merely CONTAINS one of those tags is NOT — because the + cheap version of this filter is a substring search, and a substring search + silences an operator who pastes a transcript or asks a question about a + tag by name. The filter reads the FIRST token only. + +The corpus below is drawn from real transcript shapes rather than invented, +which is the reason the whitespace and trailing-`` cases are +here: both occur. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +DEFS = ROOT / "plugin" / "hooks" / "scribe_defs.sh" +HOOK = ROOT / "plugin" / "hooks" / "scribe_autoinject.sh" + +# Written by the client. None of these is a question anyone asked. +SYNTHETIC = [ + "\nb1oojuqu3\n/tmp/x", + "/compact", + "compact", + "focus on the hooks", + "Reconnected to plugin:scribe", + "Failed to reconnect", + "Caveat: The messages below were generated…", + " \n\nabc", # whitespace must not walk the filter +] + +# Typed by a person. Every one of these must still be retrieved against. +REAL = [ + "please merge to main", + "awesome go for 3898", + "why does show up in the telemetry?", # names the tag + "fix the hookinjected", # tag, but trailing + "This session is being continued from a previous conversation.", # see module docstring + " is rendering wrong", # a tag, but not one of the client's + "here is the transcript you asked for:\nhi", +] + + +def _skip(prompt: str) -> bool: + """Run the real predicate in the real shell — not a reimplementation.""" + if shutil.which("bash") is None: # pragma: no cover + pytest.skip("hook runtime tool 'bash' not installed") + script = f'set -uo pipefail\n. "{DEFS}"\nscribe_skip_prompt "$1" && echo SKIP || echo KEEP\n' + out = subprocess.run( + ["bash", "-c", script, "_", prompt], + capture_output=True, text=True, timeout=30, + env={"PATH": os.environ["PATH"], "HOME": os.environ.get("HOME", "/tmp")}, + ) + assert out.returncode == 0, out.stderr + verdict = out.stdout.strip() + assert verdict in {"SKIP", "KEEP"}, out.stdout + return verdict == "SKIP" + + +@pytest.mark.parametrize("prompt", SYNTHETIC, ids=lambda p: p[:28]) +def test_client_written_turns_are_skipped(prompt: str) -> None: + assert _skip(prompt), f"retrieval would fire on a machine-written turn: {prompt[:60]!r}" + + +@pytest.mark.parametrize("prompt", REAL, ids=lambda p: p[:28]) +def test_operator_prompts_survive(prompt: str) -> None: + assert not _skip(prompt), f"a real prompt was silenced: {prompt[:60]!r}" + + +def test_the_filter_is_a_prefix_test_not_a_substring_test() -> None: + """The distinction the whole design rests on, asserted as one fact. + + Stated separately from the parametrised cases because a future rewrite + that reaches for `grep -q` or `case *""*` passes nothing here, and + the parametrised failure would read as "one odd input" rather than "the + implementation changed shape". + """ + tag = "" + assert _skip(tag + " trailing") + assert not _skip("what is " + tag + "?") + + +def test_the_hook_consults_the_filter_before_spending_a_request() -> None: + """Placement, not merely presence. + + The guard is worthless below the `curl`: the point is that no row is + logged, so it has to sit ahead of the request that would log one. + """ + body = HOOK.read_text() + assert "scribe_skip_prompt" in body, "the hook does not consult the filter" + assert body.index("scribe_skip_prompt") < body.index("curl -fsS"), \ + "the filter runs after the request it exists to prevent"