Write-path semantic arm: query snippets by concept, not raw code (#86)

Implements #2242. The semantic arm now queries with what the code says it is FOR — declarations plus the first docstring / JSDoc / leading comment — instead of the raw payload, because snippet documents are prose-forward: 0.823 vs 0.743, with double the separation from the noise floor. No doc means no rewrite (a bare identifier measured 0.671, worse than the code), and the 48-char floor still judges the raw payload before the rewrite. No migration, no plugin manifest bump — server-side only.
2026-07-30 10:21:19 -04:00
parent 0550bf4687 57781770c3
commit a36837c96a
2 changed files with 304 additions and 0 deletions
@@ -98,6 +98,69 @@ WRITEPATH_DEFAULT_THRESHOLD = 0.68
 # scale. It also saves a pointless embedding round-trip on trivial edits.
 WRITEPATH_MIN_CODE_CHARS = 48

+# --- concept extraction for the semantic arm's query (#2242) ------------------
+# A snippet's embedded text is f"{title}\n{body}", and for a snippet that body is
+# composed markdown: **When to use:**, **Signature:**, **Location:**, then the
+# fenced code. So `when_to_use` — the description of what the thing is FOR —
+# appears twice in the vector, and the document is prose-forward.
+#
+# The arm used to query it with raw code and no prose at all. Measured on the
+# deployed instance against snippet #2222, same corpus:
+#   query built from            score   best unrelated   separation
+#   raw code body               0.743   0.630            0.11
+#   name + docstring            0.823   0.602            0.22
+#   hand-written concept prose  0.835   0.583            0.25
+# A 12-word description beats a near-verbatim reimplementation of the function,
+# and code-as-query RAISES the noise floor. It is also the cleanest explanation
+# for the fragment miss recorded on #2223: a short code excerpt has almost no
+# prose to match against a document that is mostly prose.
+#
+# So we send the concept instead — and shape it like a snippet's own title,
+# "{name} — {when_to_use}", because that is the form the 0.823 measurement used.
+# Undocumented code yields little, and a Vue SFC or a config file yields nothing;
+# those fall back to the raw payload and behave exactly as before. This raises
+# the ceiling for documented helpers rather than fixing every case.
+
+# Declaration forms, one pattern per shape, every pattern exposing (name, params)
+# so composition doesn't have to care which matched. Deliberately regex and not a
+# real parser: this runs on a PreToolUse hook's critical path, the payload is
+# frequently a FRAGMENT that no parser would accept (an Edit's new_string is
+# rarely a valid module), and a miss costs only a fallback to today's behaviour.
+_CONCEPT_DECL_PATTERNS = (
+    # python: def / async def, and class with optional bases
+    re.compile(r"^[ \t]*(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M),
+    re.compile(r"^[ \t]*class[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))?", re.M),
+    # js/ts: function decl, and the const-arrow form that dominates modern code
+    re.compile(r"^[ \t]*(?:export[ \t]+)?(?:default[ \t]+)?(?:async[ \t]+)?function[ \t]+([A-Za-z_$][\w$]*)[ \t]*(\([^)]*\))", re.M),
+    re.compile(r"^[ \t]*(?:export[ \t]+)?(?:const|let|var)[ \t]+([A-Za-z_$][\w$]*)[ \t]*=[ \t]*(?:async[ \t]*)?(\([^)]*\))[ \t]*=>", re.M),
+    # rust / go
+    re.compile(r"^[ \t]*(?:pub[ \t]+)?fn[ \t]+([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M),
+    re.compile(r"^[ \t]*func[ \t]+(?:\([^)]*\)[ \t]*)?([A-Za-z_]\w*)[ \t]*(\([^)]*\))", re.M),
+    # posix shell: name() {
+    re.compile(r"^[ \t]*([A-Za-z_]\w*)[ \t]*(\(\))[ \t]*\{", re.M),
+)
+
+# Doc forms, tried in order. The Python pattern also matches a triple-quoted
+# string that isn't a docstring — accepted: a stray literal is still text about
+# what the code does far more often than it's misleading, and the cost is a
+# slightly worse query rather than a wrong answer.
+_CONCEPT_PY_DOC = re.compile(r'("""|\'\'\')(.*?)\1', re.S)
+_CONCEPT_JSDOC = re.compile(r"/\*\*(.*?)\*/", re.S)
+_CONCEPT_LEADING_COMMENT = re.compile(r"\A(?:[ \t]*(?://|#)[^\n]*\n?)+")
+# A shebang is a comment to the regex above but says nothing about what the code
+# DOES, and it would otherwise open the doc with "/usr/bin/env bash".
+_CONCEPT_SHEBANG = re.compile(r"\A#![^\n]*\n")
+_CONCEPT_COMMENT_MARKER = re.compile(r"^[ \t]*(?://+|#+!?)[ \t]?", re.M)
+_CONCEPT_JSDOC_STAR = re.compile(r"^[ \t]*\*+[ \t]?", re.M)
+
+# Cap the doc so a long module docstring can't drown out the declaration, and cap
+# declarations so a 40-function Write doesn't turn into a wall of signatures.
+_CONCEPT_MAX_DOC_CHARS = 400
+_CONCEPT_MAX_DECLS = 4
+# Below this much substance the "concept" is too thin to be a better query than
+# the code itself (e.g. all we found was `f()`), so we keep the raw payload.
+_CONCEPT_MIN_CHARS = 16
+
 # Margin gate: drop any hit more than this far below the top hit's score, so a
 # single strong match doesn't drag in a wall of barely-passing neighbours.
 _AUTOINJECT_BAND = 0.10
@@ -337,6 +400,77 @@ def _prior_art_line(item: dict, marker: str, owner: str | None) -> str:
    return line


+def _concept_doc(code: str) -> str:
+    """The first doc-ish prose in `code`: docstring, else JSDoc, else leading comments."""
+    m = _CONCEPT_PY_DOC.search(code)
+    if m:
+        return _collapse(m.group(2))
+
+    m = _CONCEPT_JSDOC.search(code)
+    if m:
+        return _collapse(_CONCEPT_JSDOC_STAR.sub("", m.group(1)))
+
+    # Only a comment block at the very TOP counts. A comment further down is
+    # usually about one line of the implementation, not about the whole thing.
+    m = _CONCEPT_LEADING_COMMENT.match(_CONCEPT_SHEBANG.sub("", code))
+    if m:
+        return _collapse(_CONCEPT_COMMENT_MARKER.sub("", m.group(0)))
+
+    return ""
+
+
+def _collapse(text: str) -> str:
+    """One line, single-spaced, length-capped — embedder input, not display text."""
+    return " ".join((text or "").split())[:_CONCEPT_MAX_DOC_CHARS].strip()
+
+
+def concept_query(code: str) -> str:
+    """Rewrite a write payload as a CONCEPT query, or "" to keep the raw payload.
+
+    Returns something shaped like a snippet's own title — "name(params) — what it
+    does" — because that is the form that measured best against the prose-forward
+    snippet documents (#2242; see the table at _CONCEPT_DECL_PATTERNS).
+
+    Returns "" rather than raising or guessing whenever there's nothing worth
+    sending: no declarations and no doc, or a result too thin to beat the code it
+    would replace. The caller treats "" as "use the payload as-is", so every
+    unhandled language degrades to exactly the previous behaviour.
+    """
+    if not code or not code.strip():
+        return ""
+
+    decls: list[str] = []
+    for pattern in _CONCEPT_DECL_PATTERNS:
+        for match in pattern.finditer(code):
+            name, params = match.group(1), match.group(2) or ""
+            label = f"{name}{params}".strip()
+            if label and label not in decls:
+                decls.append(label)
+            if len(decls) >= _CONCEPT_MAX_DECLS:
+                break
+        if len(decls) >= _CONCEPT_MAX_DECLS:
+            break
+
+    doc = _concept_doc(code)
+
+    # NO DOC, NO REWRITE. An identifier alone is not a concept, and it measured
+    # WORSE than the code it would replace: `collapse_into_clusters(edges)` scored
+    # 0.671 against #2222 where the full code body scored 0.743. Separation from
+    # the noise floor is identical (0.113 either way), but the absolute value
+    # drops below the 0.68 bar — so preferring a bare name would convert a
+    # comfortable hit into a miss. Undocumented code keeps the raw payload.
+    if not doc:
+        return ""
+
+    head = ", ".join(decls)
+    query = f"{head} — {doc}" if head else doc
+
+    # Guard against a doc so terse it says nothing ("# TODO", "/** x */").
+    if len("".join(query.split())) < _CONCEPT_MIN_CHARS:
+        return ""
+    return query
+
+
 async def get_writepath_config(user_id: int) -> dict:
    """Write-path trigger settings: its own `enabled` and `threshold`, auto-inject's top_k.

@@ -451,6 +585,14 @@ async def build_write_path_hint(
    # let a deeply-nested one-liner through on padding alone.
    if len("".join(query.split())) < WRITEPATH_MIN_CODE_CHARS:
        query = ""
+    # ORDER MATTERS: the floor above judges the RAW payload, this rewrites it.
+    # Snippet documents are prose-forward, so a concept query out-scores the code
+    # itself by a wide margin (#2242). The rewritten query is allowed to be
+    # short — "slugify(t) — turn text into a url slug" is a fine query at 38
+    # chars, and it only exists because the raw payload already cleared the
+    # floor. Applying the floor after this would throw away the best queries.
+    if query:
+        query = concept_query(query) or query
    if remaining > 0 and query:
        t0 = time.perf_counter()
        hits = await semantic_search_notes(
@@ -423,6 +423,168 @@ def test_the_floor_stays_below_the_smallest_plausible_helper():
    assert pc.WRITEPATH_MIN_CODE_CHARS > len("".join("x = 1".split()))


+# --- concept extraction: query by what the code IS FOR, not by the code (#2242) ---
+
+def test_concept_query_prefers_the_name_and_docstring():
+    """Snippet documents are prose-forward — `when_to_use` appears in both the
+    title and the body — so a concept query out-scores the code itself (0.823 vs
+    0.743 measured against #2222). The shape mirrors a snippet's own title."""
+    from scribe.services.plugin_context import concept_query
+    q = concept_query(
+        'def collapse_into_clusters(edges):\n'
+        '    """Collapse similar-pairs into connected components."""\n'
+        '    parent = {}\n'
+        '    return parent\n'
+    )
+    assert q == "collapse_into_clusters(edges) — Collapse similar-pairs into connected components."
+
+
+def test_concept_query_reads_jsdoc_and_the_const_arrow_form():
+    """The dominant shape in this repo's frontend isn't `function foo()`."""
+    from scribe.services.plugin_context import concept_query
+    q = concept_query(
+        "/**\n"
+        " * Rate-limit a callback so it fires once after the last call.\n"
+        " */\n"
+        "export const debounce = (fn: Fn, wait = 250) => {\n"
+        "  let t: number | undefined;\n"
+        "};\n"
+    )
+    assert q == "debounce(fn: Fn, wait = 250) — Rate-limit a callback so it fires once after the last call."
+
+
+def test_concept_query_does_not_let_a_shebang_become_the_description():
+    """`#!` matches the leading-comment pattern but says nothing about the code."""
+    from scribe.services.plugin_context import concept_query
+    q = concept_query(
+        "#!/usr/bin/env bash\n"
+        "# Resolve the repo remote and url-encode it for the plugin API.\n"
+        "encode_remote() {\n"
+        "  git remote get-url origin\n"
+        "}\n"
+    )
+    assert q == "encode_remote() — Resolve the repo remote and url-encode it for the plugin API."
+    assert "usr/bin/env" not in q
+
+
+def test_concept_query_falls_back_when_there_is_no_doc():
+    """MEASURED RULE, not a taste call. An identifier alone scored 0.671 against
+    #2222 where the full code body scored 0.743 — same separation from the noise
+    floor (0.113), but below the 0.68 bar, so preferring the bare name would turn
+    a comfortable hit into a miss. Undocumented code keeps the raw payload."""
+    from scribe.services.plugin_context import concept_query
+    assert concept_query(
+        "def collapse_into_clusters(edges):\n"
+        "    parent = {}\n"
+        "    return parent\n"
+    ) == ""
+
+
+def test_concept_query_falls_back_on_a_doc_that_says_nothing():
+    from scribe.services.plugin_context import concept_query
+    assert concept_query('def f(x):\n    """TODO"""\n    return x\n') == ""
+
+
+def test_concept_query_falls_back_when_nothing_parses():
+    """A Vue SFC has neither a recognised declaration nor a doc block. Every
+    unhandled language must degrade to exactly the pre-#2242 behaviour."""
+    from scribe.services.plugin_context import concept_query
+    assert concept_query(
+        '<script setup lang="ts">\n'
+        "const props = defineProps<{ modelValue: string }>()\n"
+        "</script>\n"
+    ) == ""
+
+
+def test_concept_query_accepts_a_doc_without_a_declaration():
+    """A module docstring is already the concept; it needs no signature."""
+    from scribe.services.plugin_context import concept_query
+    q = concept_query('"""Turn a raw git remote into the project it belongs to."""\nMAPPING = {}\n')
+    assert q == "Turn a raw git remote into the project it belongs to."
+
+
+def test_concept_query_caps_what_it_collects():
+    """A 40-function Write must not become a wall of signatures, and a long module
+    docstring must not drown out the declaration."""
+    from scribe.services import plugin_context as pc
+    many = '"""Utilities."""\n' + "".join(
+        f"def helper_{i}(a, b):\n    return a\n" for i in range(12)
+    )
+    q = pc.concept_query(many)
+    assert q.count("helper_") == pc._CONCEPT_MAX_DECLS
+
+    long_doc = '"""' + ("word " * 400) + '"""\ndef f(a):\n    return a\n'
+    assert len(pc.concept_query(long_doc)) <= pc._CONCEPT_MAX_DOC_CHARS + 64
+
+
+@pytest.mark.asyncio
+async def test_semantic_arm_queries_the_concept_not_the_raw_payload():
+    from scribe.services import plugin_context as pc
+    search = AsyncMock(return_value=[])
+    with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
+         patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
+         patch.object(pc, "semantic_search_notes", search), \
+         patch.object(pc, "record_retrieval", MagicMock()):
+        await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE)
+    sent = search.await_args.args[1] if len(search.await_args.args) > 1 else search.await_args.kwargs["query"]
+    assert sent.startswith("debounce(fn, wait=0.25)")
+    assert "Rate-limit a callback so it fires once after the last call." in sent
+    assert "nonlocal timer" not in sent   # the implementation body is gone
+
+
+@pytest.mark.asyncio
+async def test_telemetry_records_the_query_actually_sent():
+    """retrieval_logs is what the threshold gets tuned from, so it has to hold the
+    concept query — logging the raw code would make the scores uninterpretable."""
+    from scribe.services import plugin_context as pc
+    rec = MagicMock()
+    with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
+         patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
+         patch.object(pc, "semantic_search_notes", AsyncMock(return_value=[])), \
+         patch.object(pc, "record_retrieval", rec):
+        await pc.build_write_path_hint(1, "src/x.py", code=REAL_CODE)
+    logged = rec.call_args.kwargs["query"]
+    assert logged.startswith("debounce(fn, wait=0.25)")
+    assert "nonlocal timer" not in logged
+
+
+@pytest.mark.asyncio
+async def test_the_floor_judges_the_raw_payload_not_the_concept():
+    """ORDER OF OPERATIONS. The floor asks "is this a helper being written?" of the
+    RAW payload; the rewrite happens after. A concept query is allowed to be
+    shorter than the floor — that's the whole point, since the best queries are
+    short — but a sub-floor payload must stay silent even when it has a docstring."""
+    from scribe.services import plugin_context as pc
+
+    # Clears the floor, and its concept is deliberately SHORTER than the floor.
+    verbose = (
+        'def slugify(text):\n'
+        '    """Turn text into a url slug."""\n'
+        '    out = re.sub(r"[^a-z0-9]+", "-", text.lower())\n'
+        '    return out.strip("-")\n'
+    )
+    search = AsyncMock(return_value=[])
+    with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
+         patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
+         patch.object(pc, "semantic_search_notes", search), \
+         patch.object(pc, "record_retrieval", MagicMock()):
+        await pc.build_write_path_hint(1, "src/x.py", code=verbose)
+    concept = pc.concept_query(verbose)
+    assert len("".join(concept.split())) < pc.WRITEPATH_MIN_CODE_CHARS
+    search.assert_called_once()   # ...and it still searched
+
+    # Under the floor, docstring notwithstanding.
+    tiny = 'def f(x):\n    """Slug it."""\n    return x\n'
+    search2 = AsyncMock(return_value=[])
+    with patch.object(pc, "get_writepath_config", AsyncMock(return_value=_cfg())), \
+         patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
+         patch.object(pc, "semantic_search_notes", search2), \
+         patch.object(pc, "record_retrieval", MagicMock()):
+        await pc.build_write_path_hint(1, "src/x.py", code=tiny)
+    assert len("".join(tiny.split())) < pc.WRITEPATH_MIN_CODE_CHARS
+    search2.assert_not_called()
+
+
 # --- the route between them --------------------------------------------------

 def test_route_reads_every_arg_the_hook_sends():