The hooks need no jq, and the prompt boundary retrieves against prompts #171

Merged
bvandeusen merged 4 commits from dev into main 2026-09-20 18:50:03 -04:00
4 changed files with 169 additions and 1 deletions
Showing only changes of commit 1f7ff7b215 - Show all commits
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "scribe",
"description": "Scribe for Claude Code: connects the scribe MCP server, adds the hooks that deliver live project state and relevant records at the right moment, ships the shared client-neutral Scribe skills (using-scribe, writing-plans, reporting-back, systematic-debugging, verification, brainstorming, reusing-code, shape-accounting), and syncs your saved Scribe Processes as skills (/scribe:sync).",
"version": "2026.09.20.1623",
"version": "2026.09.20.2244",
"author": {
"name": "Bryan Van Deusen"
},
+8
View File
@@ -53,6 +53,14 @@ event_cwd=$(scribe_json_pick "$event_flat" '.cwd')
# Nothing to retrieve against.
[ -n "$prompt" ] || exit 0
# A turn CLAUDE CODE wrote, not the operator (#4200) — a task notification, a
# slash-command echo, a local command's caveat banner. Leaving before the
# request matters more for the LOG than for the noise: a retrieval call
# recorded against a notification inflates this surface's denominator and
# seeds `near_misses` with demand nobody expressed. scribe_skip_prompt carries
# the measurement.
scribe_skip_prompt "$prompt" && exit 0
# Unconfigured install → silent (auto-inject is pure enrichment).
scribe_config || exit 0
+43
View File
@@ -8,6 +8,8 @@
# same three things, kept here so they cannot drift apart:
#
# scribe_skip_path PATH formats that hold prose or data, not shapes
# scribe_skip_prompt TEXT the turn was written by the CLIENT, not the
# operator — skip it entirely (#4200)
# scribe_defs stdin code → "kind<TAB>name" per definition
# scribe_local_dups ROOT REL "kind<TAB>name" lines on stdin → the by-name
# local-duplicate lines (ARM 1, #2280)
@@ -43,6 +45,47 @@ scribe_skip_path() {
return 1
}
# Skip a turn the CLIENT wrote rather than the operator (#4200). Claude Code
# submits several kinds of machine-written text through UserPromptSubmit — a
# task notification, the echo of a slash command, the caveat banner a local
# command prints — and they reach `.prompt` indistinguishable from typed words
# unless something looks.
#
# THIS IS A TELEMETRY FIX FIRST AND A NOISE FIX SECOND, which is the part
# worth keeping straight: retrieving against a notification usually returns
# nothing, so the visible cost looks like one wasted embedding. The real cost
# is that the call is LOGGED. It inflates the denominator of every
# prompt-boundary surface, so delivery rate reads low for a reason that has
# nothing to do with retrieval; and the refusal lands in `near_misses`, where
# a later tuning decision reads it as unmet demand. Measured on #3898: 15 of
# the top 20 preference near-misses were `<task-notification>` blocks, ALL
# matching one record, ALL within thousandths of the floor. A floor lowered to
# serve that apparent demand would inject that record into every notification
# — the instrument arguing for the wrong fix, which is #379's lesson again.
#
# A PREFIX TEST, NOT A SUBSTRING ONE, and that distinction is the whole safety
# argument. A real prompt may well CONTAIN one of these tags — an operator
# pasting a transcript, or a `<system-reminder>` appended after typed words —
# and must still be retrieved against. Nothing an operator types BEGINS with a
# client envelope.
#
# Leading whitespace goes first: the tag is what identifies the turn, and one
# stray newline ahead of it would otherwise walk past the entire filter.
#
# NOT LISTED, ON PURPOSE: the compaction-resume injection. It is machine
# written too, but it is a summary of real work rather than plumbing, and a
# resumed session is exactly where recalling a rule earns its keep.
scribe_skip_prompt() {
_scribe_prompt=${1#"${1%%[![:space:]]*}"}
case "$_scribe_prompt" in
"<task-notification>"*|\
"<command-name>"*|"<command-message>"*|"<command-args>"*|\
"<local-command-stdout>"*|"<local-command-stderr>"*|"<local-command-caveat>"*)
return 0 ;;
esac
return 1
}
# ---------------------------------------------------------------------------
# JSON AND URL-ENCODING, WITHOUT jq (#4107).
#
+117
View File
@@ -0,0 +1,117 @@
"""The prompt boundary retrieves against PROMPTS, not against plumbing (#4200).
Claude Code submits more than typed words through `UserPromptSubmit`. A task
notification, the echo of a slash command and the caveat banner a local command
prints all arrive as user turns, and all reach `.prompt` looking exactly like
something the operator wrote. `scribe_autoinject.sh` believed them.
WHY THIS IS TESTED AT ALL, given that retrieving against a notification almost
never returns anything. The cost that matters is not the wasted embedding, it
is the LOG ROW. Every one of those calls counts in the denominator of every
prompt-boundary surface, so delivery rate reads low for a reason unrelated to
retrieval quality; and each refusal lands in `near_misses`, where a later
tuning decision reads it as demand. Measured on #3898: 15 of the top 20
preference near-misses were `<task-notification>` blocks, all matching one
record, all within thousandths of the floor. A floor lowered to serve that
apparent demand would have injected that record into every notification — an
instrument arguing for the wrong fix, which is exactly #379.
So the assertions split in two, and the SECOND half is the one guarding
against the obvious wrong implementation:
* the client's envelopes are skipped;
* a real prompt that merely CONTAINS one of those tags is NOT — because the
cheap version of this filter is a substring search, and a substring search
silences an operator who pastes a transcript or asks a question about a
tag by name. The filter reads the FIRST token only.
The corpus below is drawn from real transcript shapes rather than invented,
which is the reason the whitespace and trailing-`<system-reminder>` cases are
here: both occur.
"""
from __future__ import annotations
import os
import shutil
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
DEFS = ROOT / "plugin" / "hooks" / "scribe_defs.sh"
HOOK = ROOT / "plugin" / "hooks" / "scribe_autoinject.sh"
# Written by the client. None of these is a question anyone asked.
SYNTHETIC = [
"<task-notification>\n<task-id>b1oojuqu3</task-id>\n<output-file>/tmp/x</output-file>",
"<command-name>/compact</command-name>",
"<command-message>compact</command-message>",
"<command-args>focus on the hooks</command-args>",
"<local-command-stdout>Reconnected to plugin:scribe</local-command-stdout>",
"<local-command-stderr>Failed to reconnect</local-command-stderr>",
"<local-command-caveat>Caveat: The messages below were generated…</local-command-caveat>",
" \n<task-notification>\n<task-id>abc</task-id>", # whitespace must not walk the filter
]
# Typed by a person. Every one of these must still be retrieved against.
REAL = [
"please merge to main",
"awesome go for 3898",
"why does <task-notification> show up in the telemetry?", # names the tag
"fix the hook<system-reminder>injected</system-reminder>", # tag, but trailing
"This session is being continued from a previous conversation.", # see module docstring
"<taskbar> is rendering wrong", # a tag, but not one of the client's
"here is the transcript you asked for:\n<local-command-stdout>hi</local-command-stdout>",
]
def _skip(prompt: str) -> bool:
"""Run the real predicate in the real shell — not a reimplementation."""
if shutil.which("bash") is None: # pragma: no cover
pytest.skip("hook runtime tool 'bash' not installed")
script = f'set -uo pipefail\n. "{DEFS}"\nscribe_skip_prompt "$1" && echo SKIP || echo KEEP\n'
out = subprocess.run(
["bash", "-c", script, "_", prompt],
capture_output=True, text=True, timeout=30,
env={"PATH": os.environ["PATH"], "HOME": os.environ.get("HOME", "/tmp")},
)
assert out.returncode == 0, out.stderr
verdict = out.stdout.strip()
assert verdict in {"SKIP", "KEEP"}, out.stdout
return verdict == "SKIP"
@pytest.mark.parametrize("prompt", SYNTHETIC, ids=lambda p: p[:28])
def test_client_written_turns_are_skipped(prompt: str) -> None:
assert _skip(prompt), f"retrieval would fire on a machine-written turn: {prompt[:60]!r}"
@pytest.mark.parametrize("prompt", REAL, ids=lambda p: p[:28])
def test_operator_prompts_survive(prompt: str) -> None:
assert not _skip(prompt), f"a real prompt was silenced: {prompt[:60]!r}"
def test_the_filter_is_a_prefix_test_not_a_substring_test() -> None:
"""The distinction the whole design rests on, asserted as one fact.
Stated separately from the parametrised cases because a future rewrite
that reaches for `grep -q` or `case *"<tag>"*` passes nothing here, and
the parametrised failure would read as "one odd input" rather than "the
implementation changed shape".
"""
tag = "<task-notification>"
assert _skip(tag + " trailing")
assert not _skip("what is " + tag + "?")
def test_the_hook_consults_the_filter_before_spending_a_request() -> None:
"""Placement, not merely presence.
The guard is worthless below the `curl`: the point is that no row is
logged, so it has to sit ahead of the request that would log one.
"""
body = HOOK.read_text()
assert "scribe_skip_prompt" in body, "the hook does not consult the filter"
assert body.index("scribe_skip_prompt") < body.index("curl -fsS"), \
"the filter runs after the request it exists to prevent"