From 57781770c349c3995e764daf509337c3618673bb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 30 Jul 2026 09:58:06 -0400 Subject: [PATCH] feat(write-path): query snippets by concept, not by raw code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements #2242, from the operator's question: would it make more sense to search by the concept of the snippet than by the code itself? It would, measurably. A snippet's embedded text is f"{title}\n{body}", and a snippet's body is composed markdown — When to use / Signature / Location, then the fenced code — so `when_to_use` appears TWICE in the vector and the document is prose-forward. The semantic arm was interrogating it with raw code carrying no prose at all. Measured on the deployed instance against snippet #2222: query built from score best unrelated separation raw code body 0.743 0.630 0.11 name + docstring 0.823 0.602 0.22 hand-written concept prose 0.835 0.583 0.25 A 12-word description beats a near-verbatim reimplementation of the function, and code-as-query RAISES the noise floor. It's also the cleanest explanation for the fragment miss recorded on #2223: a short excerpt carries almost no prose to match a document that is mostly prose. So build the query from what the code says it's FOR — declarations plus the first docstring / JSDoc / leading comment block — shaped as "name(params) — what it does", mirroring a snippet's own title, which is the form that measured 0.823. Server-side rather than in the hook: no manifest bump, so installed 0.1.20 plugins get this immediately; multi-language parsing in bash would be miserable; and it's unit-testable here. Two rules worth calling out, both measured rather than chosen: - NO DOC, NO REWRITE. A bare identifier is not a concept and scored 0.671 vs the code body's 0.743. Separation from noise is identical either way (0.113), but the absolute drops under the 0.68 bar, so preferring a bare name would convert a comfortable hit into a miss. Undocumented code keeps the raw payload. - The 48-char floor still judges the RAW payload, before the rewrite. A concept query is allowed to be shorter than the floor — that is the point, the best queries are short — but a sub-floor edit stays silent even with a docstring. Applying the floor after extraction would discard the best queries. Regex, not a parser: this is on a PreToolUse critical path and an Edit's new_string is rarely a valid module, so a miss must cost only a fallback. Every unrecognised language (Vue SFC, config files) degrades to exactly the previous behaviour. Telemetry now logs the concept query rather than the code, since retrieval_logs is what the threshold gets tuned from and the two aren't comparable. 0.68 is left alone: signal rises to 0.82 while noise FALLS to 0.58, so the bar sits mid-gap instead of near the edge. To be re-measured against the deployed instance rather than assumed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs --- src/scribe/services/plugin_context.py | 142 ++++++++++++++++++++++ tests/test_write_path_trigger.py | 162 ++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) diff --git a/src/scribe/services/plugin_context.py b/src/scribe/services/plugin_context.py index fdf04d8..708a278 100644 --- a/src/scribe/services/plugin_context.py +++ b/src/scribe/services/plugin_context.py @@ -98,6 +98,69 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68 # scale. It also saves a pointless embedding round-trip on trivial edits. WRITEPATH_MIN_CODE_CHARS = 48 +# --- concept extraction for the semantic arm's query (#2242) ------------------ +# A snippet's embedded text is f"{title}\n{body}", and for a snippet that body is +# composed markdown: **When to use:**, **Signature:**, **Location:**, then the +# fenced code. So `when_to_use` — the description of what the thing is FOR — +# appears twice in the vector, and the document is prose-forward. +# +# The arm used to query it with raw code and no prose at all. Measured on the +# deployed instance against snippet #2222, same corpus: +# query built from score best unrelated separation +# raw code body 0.743 0.630 0.11 +# name + docstring 0.823 0.602 0.22 +# hand-written concept prose 0.835 0.583 0.25 +# A 12-word description beats a near-verbatim reimplementation of the function, +# and code-as-query RAISES the noise floor. It is also the cleanest explanation +# for the fragment miss recorded on #2223: a short code excerpt has almost no +# prose to match against a document that is mostly prose. +# +# So we send the concept instead — and shape it like a snippet's own title, +# "{name} — {when_to_use}", because that is the form the 0.823 measurement used. +# Undocumented code yields little, and a Vue SFC or a config file yields nothing; +# those fall back to the raw payload and behave exactly as before. This raises +# the ceiling for documented helpers rather than fixing every case. + +# Declaration forms, one pattern per shape, every pattern exposing (name, params) +# so composition doesn't have to care which matched. Deliberately regex and not a +# real parser: this runs on a PreToolUse hook's critical path, the payload is +# frequently a FRAGMENT that no parser would accept (an Edit's new_string is +# rarely a valid module), and a miss costs only a fallback to today's behaviour. +_CONCEPT_DECL_PATTERNS = ( + # python: def / async def, and class with optional bases + re.compile(r"^[ \t]*(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M), + re.compile(r"^[ \t]*class[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))?", re.M), + # js/ts: function decl, and the const-arrow form that dominates modern code + re.compile(r"^[ \t]*(?:export[ \t]+)?(?:default[ \t]+)?(?:async[ \t]+)?function[ \t]+([A-Za-z_$][\w$]*)[ \t]*(\([^)]*\))", re.M), + re.compile(r"^[ \t]*(?:export[ \t]+)?(?:const|let|var)[ \t]+([A-Za-z_$][\w$]*)[ \t]*=[ \t]*(?:async[ \t]*)?(\([^)]*\))[ \t]*=>", re.M), + # rust / go + re.compile(r"^[ \t]*(?:pub[ \t]+)?fn[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M), + re.compile(r"^[ \t]*func[ \t]+(?:\([^)]*\)[ \t]*)?([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M), + # posix shell: name() { + re.compile(r"^[ \t]*([A-Za-z_]\w*)[ \t]*(\(\))[ \t]*\{", re.M), +) + +# Doc forms, tried in order. The Python pattern also matches a triple-quoted +# string that isn't a docstring — accepted: a stray literal is still text about +# what the code does far more often than it's misleading, and the cost is a +# slightly worse query rather than a wrong answer. +_CONCEPT_PY_DOC = re.compile(r'("""|\'\'\')(.*?)\1', re.S) +_CONCEPT_JSDOC = re.compile(r"/\*\*(.*?)\*/", re.S) +_CONCEPT_LEADING_COMMENT = re.compile(r"\A(?:[ \t]*(?://|#)[^\n]*\n?)+") +# A shebang is a comment to the regex above but says nothing about what the code +# DOES, and it would otherwise open the doc with "/usr/bin/env bash". +_CONCEPT_SHEBANG = re.compile(r"\A#![^\n]*\n") +_CONCEPT_COMMENT_MARKER = re.compile(r"^[ \t]*(?://+|#+!?)[ \t]?", re.M) +_CONCEPT_JSDOC_STAR = re.compile(r"^[ \t]*\*+[ \t]?", re.M) + +# Cap the doc so a long module docstring can't drown out the declaration, and cap +# declarations so a 40-function Write doesn't turn into a wall of signatures. +_CONCEPT_MAX_DOC_CHARS = 400 +_CONCEPT_MAX_DECLS = 4 +# Below this much substance the "concept" is too thin to be a better query than +# the code itself (e.g. all we found was `f()`), so we keep the raw payload. +_CONCEPT_MIN_CHARS = 16 + # Margin gate: drop any hit more than this far below the top hit's score, so a # single strong match doesn't drag in a wall of barely-passing neighbours. _AUTOINJECT_BAND = 0.10 @@ -337,6 +400,77 @@ def _prior_art_line(item: dict, marker: str, owner: str | None) -> str: return line +def _concept_doc(code: str) -> str: + """The first doc-ish prose in `code`: docstring, else JSDoc, else leading comments.""" + m = _CONCEPT_PY_DOC.search(code) + if m: + return _collapse(m.group(2)) + + m = _CONCEPT_JSDOC.search(code) + if m: + return _collapse(_CONCEPT_JSDOC_STAR.sub("", m.group(1))) + + # Only a comment block at the very TOP counts. A comment further down is + # usually about one line of the implementation, not about the whole thing. + m = _CONCEPT_LEADING_COMMENT.match(_CONCEPT_SHEBANG.sub("", code)) + if m: + return _collapse(_CONCEPT_COMMENT_MARKER.sub("", m.group(0))) + + return "" + + +def _collapse(text: str) -> str: + """One line, single-spaced, length-capped — embedder input, not display text.""" + return " ".join((text or "").split())[:_CONCEPT_MAX_DOC_CHARS].strip() + + +def concept_query(code: str) -> str: + """Rewrite a write payload as a CONCEPT query, or "" to keep the raw payload. + + Returns something shaped like a snippet's own title — "name(params) — what it + does" — because that is the form that measured best against the prose-forward + snippet documents (#2242; see the table at _CONCEPT_DECL_PATTERNS). + + Returns "" rather than raising or guessing whenever there's nothing worth + sending: no declarations and no doc, or a result too thin to beat the code it + would replace. The caller treats "" as "use the payload as-is", so every + unhandled language degrades to exactly the previous behaviour. + """ + if not code or not code.strip(): + return "" + + decls: list[str] = [] + for pattern in _CONCEPT_DECL_PATTERNS: + for match in pattern.finditer(code): + name, params = match.group(1), match.group(2) or "" + label = f"{name}{params}".strip() + if label and label not in decls: + decls.append(label) + if len(decls) >= _CONCEPT_MAX_DECLS: + break + if len(decls) >= _CONCEPT_MAX_DECLS: + break + + doc = _concept_doc(code) + + # NO DOC, NO REWRITE. An identifier alone is not a concept, and it measured + # WORSE than the code it would replace: `collapse_into_clusters(edges)` scored + # 0.671 against #2222 where the full code body scored 0.743. Separation from + # the noise floor is identical (0.113 either way), but the absolute value + # drops below the 0.68 bar — so preferring a bare name would convert a + # comfortable hit into a miss. Undocumented code keeps the raw payload. + if not doc: + return "" + + head = ", ".join(decls) + query = f"{head} — {doc}" if head else doc + + # Guard against a doc so terse it says nothing ("# TODO", "/** x */"). + if len("".join(query.split())) < _CONCEPT_MIN_CHARS: + return "" + return query + + async def get_writepath_config(user_id: int) -> dict: """Write-path trigger settings: its own `enabled` and `threshold`, auto-inject's top_k. @@ -451,6 +585,14 @@ async def build_write_path_hint( # let a deeply-nested one-liner through on padding alone. if len("".join(query.split())) < WRITEPATH_MIN_CODE_CHARS: query = "" + # ORDER MATTERS: the floor above judges the RAW payload, this rewrites it. + # Snippet documents are prose-forward, so a concept query out-scores the code + # itself by a wide margin (#2242). The rewritten query is allowed to be + # short — "slugify(t) — turn text into a url slug" is a fine query at 38 + # chars, and it only exists because the raw payload already cleared the + # floor. Applying the floor after this would throw away the best queries. + if query: + query = concept_query(query) or query if remaining > 0 and query: t0 = time.perf_counter() hits = await semantic_search_notes( diff --git a/tests/test_write_path_trigger.py b/tests/test_write_path_trigger.py index 852a281..2cfd2d7 100644 --- a/tests/test_write_path_trigger.py +++ b/tests/test_write_path_trigger.py @@ -423,6 +423,168 @@ def test_the_floor_stays_below_the_smallest_plausible_helper(): assert pc.WRITEPATH_MIN_CODE_CHARS > len("".join("x = 1".split())) +# --- concept extraction: query by what the code IS FOR, not by the code (#2242) --- + +def test_concept_query_prefers_the_name_and_docstring(): + """Snippet documents are prose-forward — `when_to_use` appears in both the + title and the body — so a concept query out-scores the code itself (0.823 vs + 0.743 measured against #2222). The shape mirrors a snippet's own title.""" + from scribe.services.plugin_context import concept_query + q = concept_query( + 'def collapse_into_clusters(edges):\n' + ' """Collapse similar-pairs into connected components."""\n' + ' parent = {}\n' + ' return parent\n' + ) + assert q == "collapse_into_clusters(edges) — Collapse similar-pairs into connected components." + + +def test_concept_query_reads_jsdoc_and_the_const_arrow_form(): + """The dominant shape in this repo's frontend isn't `function foo()`.""" + from scribe.services.plugin_context import concept_query + q = concept_query( + "/**\n" + " * Rate-limit a callback so it fires once after the last call.\n" + " */\n" + "export const debounce = (fn: Fn, wait = 250) => {\n" + " let t: number | undefined;\n" + "};\n" + ) + assert q == "debounce(fn: Fn, wait = 250) — Rate-limit a callback so it fires once after the last call." + + +def test_concept_query_does_not_let_a_shebang_become_the_description(): + """`#!` matches the leading-comment pattern but says nothing about the code.""" + from scribe.services.plugin_context import concept_query + q = concept_query( + "#!/usr/bin/env bash\n" + "# Resolve the repo remote and url-encode it for the plugin API.\n" + "encode_remote() {\n" + " git remote get-url origin\n" + "}\n" + ) + assert q == "encode_remote() — Resolve the repo remote and url-encode it for the plugin API." + assert "usr/bin/env" not in q + + +def test_concept_query_falls_back_when_there_is_no_doc(): + """MEASURED RULE, not a taste call. An identifier alone scored 0.671 against + #2222 where the full code body scored 0.743 — same separation from the noise + floor (0.113), but below the 0.68 bar, so preferring the bare name would turn + a comfortable hit into a miss. Undocumented code keeps the raw payload.""" + from scribe.services.plugin_context import concept_query + assert concept_query( + "def collapse_into_clusters(edges):\n" + " parent = {}\n" + " return parent\n" + ) == "" + + +def test_concept_query_falls_back_on_a_doc_that_says_nothing(): + from scribe.services.plugin_context import concept_query + assert concept_query('def f(x):\n """TODO"""\n return x\n') == "" + + +def test_concept_query_falls_back_when_nothing_parses(): + """A Vue SFC has neither a recognised declaration nor a doc block. Every + unhandled language must degrade to exactly the pre-#2242 behaviour.""" + from scribe.services.plugin_context import concept_query + assert concept_query( + '\n" + ) == "" + + +def test_concept_query_accepts_a_doc_without_a_declaration(): + """A module docstring is already the concept; it needs no signature.""" + from scribe.services.plugin_context import concept_query + q = concept_query('"""Turn a raw git remote into the project it belongs to."""\nMAPPING = {}\n') + assert q == "Turn a raw git remote into the project it belongs to." + + +def test_concept_query_caps_what_it_collects(): + """A 40-function Write must not become a wall of signatures, and a long module + docstring must not drown out the declaration.""" + from scribe.services import plugin_context as pc + many = '"""Utilities."""\n' + "".join( + f"def helper_{i}(a, b):\n return a\n" for i in range(12) + ) + q = pc.concept_query(many) + assert q.count("helper_") == pc._CONCEPT_MAX_DECLS + + long_doc = '"""' + ("word " * 400) + '"""\ndef f(a):\n return a\n' + assert len(pc.concept_query(long_doc)) <= pc._CONCEPT_MAX_DOC_CHARS + 64 + + +@pytest.mark.asyncio +async def test_semantic_arm_queries_the_concept_not_the_raw_payload(): + from scribe.services import plugin_context as pc + search = AsyncMock(return_value=[]) + with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \ + patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \ + patch.object(pc, "semantic_search_notes", search), \ + patch.object(pc, "record_retrieval", MagicMock()): + await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE) + sent = search.await_args.args[1] if len(search.await_args.args) > 1 else search.await_args.kwargs["query"] + assert sent.startswith("debounce(fn, wait=0.25)") + assert "Rate-limit a callback so it fires once after the last call." in sent + assert "nonlocal timer" not in sent # the implementation body is gone + + +@pytest.mark.asyncio +async def test_telemetry_records_the_query_actually_sent(): + """retrieval_logs is what the threshold gets tuned from, so it has to hold the + concept query — logging the raw code would make the scores uninterpretable.""" + from scribe.services import plugin_context as pc + rec = MagicMock() + with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \ + patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \ + patch.object(pc, "semantic_search_notes", AsyncMock(return_value=[])), \ + patch.object(pc, "record_retrieval", rec): + await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE) + logged = rec.call_args.kwargs["query"] + assert logged.startswith("debounce(fn, wait=0.25)") + assert "nonlocal timer" not in logged + + +@pytest.mark.asyncio +async def test_the_floor_judges_the_raw_payload_not_the_concept(): + """ORDER OF OPERATIONS. The floor asks "is this a helper being written?" of the + RAW payload; the rewrite happens after. A concept query is allowed to be + shorter than the floor — that's the whole point, since the best queries are + short — but a sub-floor payload must stay silent even when it has a docstring.""" + from scribe.services import plugin_context as pc + + # Clears the floor, and its concept is deliberately SHORTER than the floor. + verbose = ( + 'def slugify(text):\n' + ' """Turn text into a url slug."""\n' + ' out = re.sub(r"[^a-z0-9]+", "-", text.lower())\n' + ' return out.strip("-")\n' + ) + search = AsyncMock(return_value=[]) + with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \ + patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \ + patch.object(pc, "semantic_search_notes", search), \ + patch.object(pc, "record_retrieval", MagicMock()): + await pc.build_write_path_hint(1, "src/x.py", code=verbose) + concept = pc.concept_query(verbose) + assert len("".join(concept.split())) < pc.WRITEPATH_MIN_CODE_CHARS + search.assert_called_once() # ...and it still searched + + # Under the floor, docstring notwithstanding. + tiny = 'def f(x):\n """Slug it."""\n return x\n' + search2 = AsyncMock(return_value=[]) + with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \ + patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \ + patch.object(pc, "semantic_search_notes", search2), \ + patch.object(pc, "record_retrieval", MagicMock()): + await pc.build_write_path_hint(1, "src/x.py", code=tiny) + assert len("".join(tiny.split())) < pc.WRITEPATH_MIN_CODE_CHARS + search2.assert_not_called() + + # --- the route between them -------------------------------------------------- def test_route_reads_every_arg_the_hook_sends():