FabledScribe/plugin/hooks/scribe_json.awk

# Scribe plugin — JSON, read without jq (#4107).
#
# WHY THIS FILE EXISTS. Every hook here parsed JSON with `jq`, and every hook
# opened with `command -v jq >/dev/null 2>&1 || exit 0`. jq is not installed by
# default on macOS, on Debian/Ubuntu base images, on Alpine, or in most CI
# containers — so on a machine without it the operator got no session context,
# no rules, no prior art and no process sync, and not one word saying why. That
# is not a missing prerequisite to document; it is the plugin pushing its own
# packaging problem onto whoever installs it. awk is POSIX: it is present as
# gawk, mawk, nawk or busybox awk everywhere a shell is.
#
# WHAT IT DOES. Reads JSON on stdin, writes one line per SCALAR:
#
#     IDX <TAB> PATH <TAB> VALUE
#
#     1	.prompt	write the tests first
#     1	.tool_input.file_path	/src/app.py
#     1	.note_ids[0]	4107
#     1	.note_ids[#]	3
#
# IDX   1-based index of the top-level JSON value (always 1 unless mode=lines).
# PATH  dot/bracket path. Array indices are 0-based.
# VALUE for a string, the RAW JSON-escaped body with the quotes stripped — not
#       the decoded text. Decoding here would let a newline or a tab inside a
#       value break the line format that makes this readable from shell. The
#       caller decodes what it actually uses, with `scribe_json_unescape`.
#       Numbers, `true`, `false` and `null` are emitted literally.
#
# EVERY ARRAY ALSO EMITS `PATH[#]` with its LENGTH, including an empty one —
# which is what makes "the server answered with zero notes" distinguishable
# from "the server did not answer", a distinction #2932 built a whole outage
# marker around. An empty object emits `PATH{#}` 0 for the same reason.
#
# TWO MODES, because the two jobs have opposite failure behaviour:
#
#   mode=whole  (default) the entire input is ONE JSON value, possibly spread
#               over many lines. A parse error produces no output at all.
#               This is a hook event on stdin, or a server response body.
#
#   mode=lines  the input is JSONL — one JSON value per line — and a line that
#               does not parse is DROPPED, the rest still read. This is the
#               transcript the Stop hooks read, where the window starts
#               mid-record by construction: `tail -n 3000` cuts wherever it
#               cuts, and the first line is routinely half a record. It is
#               exactly the `map(try fromjson catch empty)` the jq program it
#               replaces opened with. IDX counts lines that PARSED, so the
#               ordering a turn is bounded by is unaffected by the dropped one.
#
# A KEY CONTAINING `.` OR `[` WOULD MAKE AN AMBIGUOUS PATH. Nothing in the two
# JSON dialects this reads — Claude Code hook events and Scribe's own API — has
# one, and inventing an escaping scheme for a case neither producer can emit
# would cost every caller a decode for nothing. Stated so the next person hits
# a comment rather than a mystery.

function skipws(   c) {
  while (pos <= n) {
    c = substr(s, pos, 1)
    if (c == " " || c == "\t" || c == "\n" || c == "\r") pos++
    else return
  }
}

# Values are emitted one per line, so a literal control character inside a
# string — invalid JSON, but producers emit it — must not become a line break
# that silently splits one value into two records.
function emit(path, val) {
  ocount++
  opath[ocount] = path
  oval[ocount] = tame(val)
}

function tame(v) {
  if (index(v, "\n")) v = rep(v, "\n", "\\n")
  if (index(v, "\r")) v = rep(v, "\r", "\\r")
  if (index(v, "\t")) v = rep(v, "\t", "\\t")
  return v
}

# Replace every occurrence of a literal separator, via split/join rather than
# gsub: gsub's replacement string treats `\` and `&` as metacharacters, so
# emitting a literal backslash through it needs a quadruple-escape whose
# meaning varies by awk. Concatenation has no such reading.
function rep(str, sep, with,   parts, cnt, i, o) {
  cnt = split(str, parts, sep)
  o = parts[1]
  for (i = 2; i <= cnt; i++) o = o with parts[i]
  return o
}

function parse_value(path,   c) {
  if (failed) return
  skipws()
  if (pos > n) { failed = 1; return }
  c = substr(s, pos, 1)
  if (c == "{") { parse_object(path); return }
  if (c == "[") { parse_array(path); return }
  if (c == "\"") { emit(path, parse_string()); return }
  parse_literal(path)
}

# HOW FAR AHEAD A TOKEN IS READ, and why there is a limit at all.
#
# awk has no way to match a regex STARTING AT AN OFFSET, so reading a token
# means copying the rest of the buffer and anchoring with `^`. Do that per
# token and the cost is quadratic in the length of the line: a transcript
# record carrying a 400KB tool result has a hundred small tokens after it, and
# each one copied the whole 400KB again. Measured: ten such lines, 500KB in
# total, took 2.2s — while 291 ordinary lines totalling 684KB took 0.38s.
#
# So an ordinary token is read out of a WINDOW, and only a token that does not
# fit in one pays for the whole remainder — once, for itself, rather than once
# for every token that follows it. 1024 is where the curve flattens, measured
# over a real 1.19MB turn: 8192 → 800ms, 2048 → 425ms, 1024 → 351ms, 256 →
# 315ms, every one of them producing byte-identical output. Below 1024 the
# gain stops paying for the long values that then miss the window.
#
# A MATCH INSIDE THE WINDOW IS ALWAYS THE TRUE TOKEN, which is what makes this
# safe rather than merely fast. Every quote inside a JSON string is escaped, so
# the `\\.` branch consumes it; an unescaped `"` can only be the real closing
# quote. A window that cuts a string short therefore yields no match at all —
# it cannot yield a WRONG one — and the wide read below handles it.
#
# SET IN `BEGIN`, and it has to be there. A bare assignment at file scope
# is not a statement to awk, it is a PATTERN — a truthy expression with no
# action — so every input record took the default action and was ECHOED to
# stdout. Nothing downstream matched the echoed line, so it cost only noise
# until a test compared this parser's output against Python's and found one
# row too many.

function parse_string(   rest, tok) {
  rest = substr(s, pos, WINDOW)
  if (match(rest, /^"([^"\\]|\\.)*"/) == 0) {
    rest = substr(s, pos)
    if (match(rest, /^"([^"\\]|\\.)*"/) == 0) { failed = 1; return "" }
  }
  tok = substr(rest, 2, RLENGTH - 2)
  pos += RLENGTH
  return tok
}

function parse_literal(path,   rest) {
  rest = substr(s, pos, WINDOW)
  if (match(rest, /^(-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?|true|false|null)/) == 0) {
    failed = 1
    return
  }
  emit(path, substr(rest, 1, RLENGTH))
  pos += RLENGTH
}

function parse_object(path,   key, c) {
  pos++
  skipws()
  if (substr(s, pos, 1) == "}") { pos++; emit(path "{#}", 0); return }
  while (1) {
    skipws()
    if (substr(s, pos, 1) != "\"") { failed = 1; return }
    key = parse_string()
    if (failed) return
    skipws()
    if (substr(s, pos, 1) != ":") { failed = 1; return }
    pos++
    parse_value(path "." key)
    if (failed) return
    skipws()
    c = substr(s, pos, 1)
    if (c == ",") { pos++; continue }
    if (c == "}") { pos++; return }
    failed = 1
    return
  }
}

function parse_array(path,   i, c) {
  pos++
  i = 0
  skipws()
  if (substr(s, pos, 1) == "]") { pos++; emit(path "[#]", 0); return }
  while (1) {
    parse_value(path "[" i "]")
    if (failed) return
    i++
    skipws()
    c = substr(s, pos, 1)
    if (c == ",") { pos++; continue }
    if (c == "]") { pos++; emit(path "[#]", i); return }
    failed = 1
    return
  }
}

# Parse the buffer in `s` as one value. Emits nothing unless it parses whole.
function run(   i) {
  n = length(s)
  pos = 1
  failed = 0
  ocount = 0
  parse_value("")
  if (!failed) { skipws(); if (pos <= n) failed = 1 }
  if (failed) { ocount = 0; return 0 }
  idx++
  for (i = 1; i <= ocount; i++) printf "%d\t%s\t%s\n", idx, opath[i], oval[i]
  ocount = 0
  return 1
}

BEGIN { idx = 0; WINDOW = 1024 }

mode == "lines" { s = $0; run(); next }

{ buf = buf $0 "\n" }

END {
  if (mode == "lines") exit 0
  s = buf
  run()
}