fix(drafter): a wrapped docstring line beginning "class AND the" defines a shape called AND (#4222)

The definition extractor is line-oriented and knows nothing about what a
line is INSIDE. A docstring that wraps onto a line starting with a keyword
announces a definition: `AND` reached a live session as a divergence prompt
asking it to justify a symbol that does not exist, and `is` reached it as a
repo-wide duplicate of four files that define nothing of the kind.

Measured, not assumed: running the extractor over every scannable file with
and without the scan differs by twenty-two phantoms. Two of them — `with`
and `nobody`, both out of the module docstring in check_dangling_styles.py —
are persisted `code_shapes` rows that have been judged. Those need no
migration: sync_shapes marks a row it no longer extracts as vanished.

`ast` would be the honest tool for .py and is not what this uses, because
the extractor is mirrored rule for rule by an awk program in the hook, awk
cannot parse Python, and a fix only one of the pair can run is the drift the
mirror exists to prevent. Both sides now run the same left-to-right scan and
blank comment and string spans to their own newlines before any matcher sees
a line. Three things the scan has to get right, each of which cost real
definitions while it was being written:

  - a string that HOLDS a marker is not a marker. `"red /* "` in
    test_design_stylesheet.py and a triple quote inside a single-quoted
    regex in plugin_context.py each ate every definition below them.
  - `#` is a colour in CSS and a comment in Python, and the extractor is
    handed no path. An alphanumeric straight after it settles it.
  - an unterminated opener blanks NOTHING. The scan rewinds past it and
    continues, so a stray marker costs one span rather than the rest of the
    file.

The comment claiming the two extractors agree has been the only thing
holding them together, and a comment cannot fail. The mirror test now RUNS
the hook's awk over the same vectors: with the old program it reports the
phantoms, which is what a guard that can fail looks like. Across all 631
scannable files in this repo the two now agree line for line.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

plugin/hooks/scribe_defs.sh

@@ -303,15 +303,118 @@ scribe_urlenc() {
 # or the one enclosing an Edit). Rule-for-rule mirrored by the server's
 # services/coverage.py extract_shapes — ledger rows are keyed by what THAT
 # sees, so the two must agree on what counts as a definition.
+#
+# THE WHOLE INPUT IS BUFFERED (#4222) so the span scan below can look ahead.
+# The matchers are line-oriented and know nothing about what a line is INSIDE:
+# a wrapped docstring beginning "class AND the …" announces a shape called
+# `AND`, which reaches the session mid-edit as a divergence prompt about a
+# symbol that does not exist. blank_spans() replaces every comment and string
+# span with its own newlines before a matcher sees a line — the same scan, in
+# the same order, as coverage.py::_blank_spans. Change one, change both.
 scribe_defs() {
   awk '
-    {
-      # CSS class definition: .name { or .name,
-      if (match($0, /^[[:space:]]*\.[A-Za-z][A-Za-z0-9_-]*[[:space:]]*[,{]/)) {
-        t = $0; sub(/^[[:space:]]*\./, "", t); sub(/[[:space:]]*[,{].*$/, "", t)
-        if (t != "") print "css\t" t; next
+    BEGIN {
+      SQ = sprintf("%c", 39)
+      SQ3 = SQ SQ SQ
+      DQ = "\""
+      DQ3 = DQ DQ DQ
+      # The only characters that can begin a span, a line comment or a
+      # string. Everything between two of them is copied in one go rather
+      # than a character at a time.
+      MARKERS = "[" DQ SQ "/#]"
+    }
+
+    # Offset just past the one-line string opening at c, or c+1 when it does
+    # not close before the end of the line — so an apostrophe in prose costs
+    # one character rather than everything up to the next quote.
+    function string_end(L, c, q,   i, n, ch) {
+      n = length(L); i = c + 1
+      while (i <= n) {
+        ch = substr(L, i, 1)
+        if (ch == "\\") { i = i + 2; continue }
+        if (ch == q) return i + 1
+        i++
       }
-      line = $0; sub(/^[[:space:]]+/, "", line)
+      return c + 1
+    }
+
+    # Does the "#" at c open a comment, or is it a CSS colour or id? An
+    # alphanumeric straight after it is #fff or #app; anything else is a
+    # comment in every language that has one.
+    function hash_comment(L, c) {
+      return substr(L, c + 1, 1) !~ /^[A-Za-z0-9]$/
+    }
+
+    # raw[1..n] -> msk[1..n] with comment and string spans emptied. Line
+    # COUNT is preserved and column positions are not; the matchers lstrip.
+    # ONLY CLOSED SPANS ARE BLANKED: an opener with no closer is rewound past
+    # and scanning resumes, so a stray marker costs one span rather than
+    # every definition below it.
+    function blank_spans(raw, n, msk,
+                         i, c, L, len, state, closer, oplen, sl, sc, sprefix,
+                         t3, t2, ch, e, k, rest, m) {
+      for (i = 1; i <= n; i++) msk[i] = ""
+      i = 1; c = 1; state = 0; closer = ""
+      while (1) {
+        while (i <= n) {
+          L = raw[i]; len = length(L)
+          if (c > len) { i++; c = 1; continue }
+          if (state) {
+            e = index(substr(L, c), closer)
+            if (e == 0) { i++; c = 1; continue }
+            c = c + e - 1 + length(closer)
+            state = 0; closer = ""
+            continue
+          }
+          rest = substr(L, c)
+          m = match(rest, MARKERS)
+          if (m == 0) { msk[i] = msk[i] rest; i++; c = 1; continue }
+          if (m > 1) {
+            msk[i] = msk[i] substr(rest, 1, m - 1)
+            c = c + m - 1
+            continue
+          }
+          t3 = substr(L, c, 3); t2 = substr(L, c, 2); ch = substr(L, c, 1)
+          if (t3 == DQ3 || t3 == SQ3) {
+            sl = i; sc = c; sprefix = msk[i]
+            state = 1; closer = t3; oplen = 3; c = c + 3
+            continue
+          }
+          if (t2 == "/*") {
+            sl = i; sc = c; sprefix = msk[i]
+            state = 1; closer = "*/"; oplen = 2; c = c + 2
+            continue
+          }
+          if (t2 == "//" || (ch == "#" && hash_comment(L, c))) {
+            # A line comment is COPIED, not blanked: its continuation lines
+            # carry their own marker, so none can read as a definition alone.
+            msk[i] = msk[i] substr(L, c)
+            i++; c = 1
+            continue
+          }
+          if (ch == DQ || ch == SQ) {
+            k = string_end(L, c, ch)
+            msk[i] = msk[i] substr(L, c, k - c)
+            c = k
+            continue
+          }
+          msk[i] = msk[i] ch
+          c++
+        }
+        if (!state) return
+        for (k = sl; k <= n; k++) msk[k] = ""
+        msk[sl] = sprefix
+        i = sl; c = sc + oplen; state = 0; closer = ""
+      }
+    }
+
+    function emit(line,   t, rest) {
+      # CSS class definition: .name { or .name,
+      if (match(line, /^[[:space:]]*\.[A-Za-z][A-Za-z0-9_-]*[[:space:]]*[,{]/)) {
+        t = line; sub(/^[[:space:]]*\./, "", t); sub(/[[:space:]]*[,{].*$/, "", t)
+        if (t != "") print "css\t" t; return
+      }
+      sub(/^[[:space:]]+/, "", line)
       # Strip leading declaration modifiers so the definition keyword is the
       # first word regardless of language (export/pub/private/suspend/...).
       sub(/^((pub(\([a-z]+\))?|export|default|private|internal|protected|public|static|suspend|async|open|sealed|data|abstract|final|inline|unsafe|extern|override)[[:space:]]+)*/, "", line)
@@ -319,7 +422,7 @@ scribe_defs() {
       if (match(line, /^func[[:space:]]*\([^)]*\)[[:space:]]*[A-Za-z_]/)) {
         t = line; sub(/^func[[:space:]]*\([^)]*\)[[:space:]]*/, "", t)
         sub(/[^A-Za-z0-9_].*$/, "", t)
-        if (t != "") print "sym\t" t; next
+        if (t != "") print "sym\t" t; return
       }
       # Keyword-announced definitions, functions and named types alike.
       # Dunders are skipped: every class defines __init__, so "already defined
@@ -333,17 +436,24 @@ scribe_defs() {
         # nothing (mirror of coverage.py, #2904).
         if (line ~ /^type[[:space:]]/) {
           rest = line; sub(/^type[[:space:]]+[A-Za-z_$][A-Za-z0-9_$]*/, "", rest)
-          if (rest !~ /[={]/) next
+          if (rest !~ /[={]/) return
         }
-        if (t != "" && t !~ /^__.*__$/) print "sym\t" t; next
+        if (t != "" && t !~ /^__.*__$/) print "sym\t" t; return
       }
       # Arrow/expression assignment: const name = (…) / let name = async (
       if (match(line, /^(const|let)[[:space:]]+[A-Za-z_$][A-Za-z0-9_$]*[[:space:]]*=[[:space:]]*(async[[:space:]]*)?[(<]/)) {
         t = line; sub(/^(const|let)[[:space:]]+/, "", t)
         sub(/[^A-Za-z0-9_$].*$/, "", t)
-        if (t != "") print "sym\t" t; next
+        if (t != "") print "sym\t" t; return
       }
     }
+
+    { raw[NR] = $0 }
+
+    END {
+      blank_spans(raw, NR, msk)
+      for (r = 1; r <= NR; r++) emit(msk[r])
+    }
   ' 2>/dev/null
 }