Files
FabledScribe/tests/test_shape_check_hook.py
T
bvandeusenandClaude Opus 5.5 f1fbdf746a
CI & Build / Python lint (push) Successful in 5s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / integration (push) Successful in 52s
CI & Build / Python tests (push) Successful in 1m38s
CI & Build / Build & push image (push) Successful in 33s
feat(shapes): the agent judges what it wrote, at the end of the turn (milestone 439 steps 1-3)
Recording used to be decided by machinery — the only "record it" prompt
fired when a same-named copy already existed (#2664), so a first instance of
a reusable piece was never asked about, and judgment arrived only through
audits. Now the question is asked where the knowledge is: the end of the
turn that wrote the code, of the agent that wrote it.

- Write hooks keep `<sid>.written.ids` (path, kind, name) for every
  definition a write names; a new file adds a `file` line for its stem — a
  candidate in any language without a framework rule (scribe_written_append).
- Stop hook scribe_shape_check.sh sends the ledger to GET
  /api/plugin/shape-check and blocks once, in the server's words, when
  anything is unjudged. Same discipline as the report check: never twice,
  never without a recorded check, another hook's loop left alone; the ledger
  is kept when the instance cannot be reached.
- shape_ledger.unjudged_shapes: no row, unclassified, scoped and hook stamps
  are unjudged; an agent/audit/import verdict is not. A snippet recorded at
  the shape answers for it until the refresh stamps it canonical.
- services/shape_check owns the reason text and records every outcome in
  app_logs (passed / blocked / judged_after_block / left_after_block).
- classify_shapes(repo=…) judges a shape the ledger has not synced yet via a
  provisional row under a bound repo; the sync confirms it, or vanishes and
  revives it with the verdict intact. An unbound repo is refused.

Plugin version minted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-01 08:39:36 -04:00

172 lines
7.2 KiB
Python

"""The end-of-turn shape check (milestone 439): the write hooks note what each
write defined, and the Stop hook asks the instance which of those nobody has
judged, blocking once in the server's words.
Runs the real shell against the shared HTTP sink. What it pins: the write
hooks' ledger (a Write of a new file adds a `file` line; a skipped path adds
nothing); silence with nothing written; a block only on a reason the instance
returned; the post-block stop recorded and let through; another hook's block
loop left alone; the ledger kept when the instance could not be reached.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from pathlib import Path
import pytest
from tests.helpers import http_sink
HOOKS = Path(__file__).resolve().parents[1] / "plugin" / "hooks"
CHECK = HOOKS / "scribe_shape_check.sh"
PRIOR_ART = HOOKS / "scribe_prior_art.sh"
REASON = "SERVER REASON: judge what you wrote"
def _env(tmp_path, url="http://127.0.0.1:9"):
for tool in ("curl", "bash", "git"):
if shutil.which(tool) is None:
pytest.skip(f"hook runtime tool {tool!r} not installed")
return {"PATH": os.environ["PATH"], "SCRIBE_URL": url, "SCRIBE_TOKEN": "t",
"TMPDIR": str(tmp_path), "HOME": str(tmp_path)}
def _ledger(tmp_path, session="s1"):
return tmp_path / "scribe-priorart" / f"{session}.written.ids"
def _write_ledger(tmp_path, lines, session="s1"):
path = _ledger(tmp_path, session)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("".join(f"{line}\n" for line in lines))
return path
def _stop(env, cwd, active=False, session="s1"):
out = subprocess.run(
["bash", str(CHECK)],
input=json.dumps({"session_id": session, "transcript_path": "/nonexistent",
"cwd": str(cwd), "hook_event_name": "Stop",
"stop_hook_active": active}),
capture_output=True, text=True, env=env, timeout=30,
)
assert out.returncode == 0, out.stderr
return out.stdout.strip()
def _repo(tmp_path):
repo = tmp_path / "repo"
repo.mkdir()
subprocess.run(["git", "init", "-q", str(repo)], check=True)
subprocess.run(["git", "-C", str(repo), "remote", "add", "origin",
"https://git.example.com/alice/widget.git"], check=True)
return repo
# ── the write hooks keep the ledger ───────────────────────────────────────
def _pre_write(env, repo, rel, content, tool="Write"):
field = "content" if tool == "Write" else "new_string"
out = subprocess.run(
["bash", str(PRIOR_ART)],
input=json.dumps({"session_id": "s1", "cwd": str(repo), "tool_name": tool,
"tool_input": {"file_path": str(repo / rel), field: content}}),
capture_output=True, text=True, env=env, timeout=30,
)
assert out.returncode == 0, out.stderr
def test_a_write_notes_what_it_defined_and_a_new_file_is_a_candidate(tmp_path):
with http_sink(b'{"note_ids":[]}') as (port, _seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
repo = _repo(tmp_path)
(repo / "web").mkdir()
_pre_write(env, repo, "web/StatusChip.svelte",
"<script>\nfunction tone(s) {\n return s;\n}\n</script>\n"
"<style>\n.chip {\n color: red;\n}\n</style>\n")
lines = _ledger(tmp_path).read_text().splitlines()
assert "web/StatusChip.svelte\tfile\tStatusChip" in lines
assert "web/StatusChip.svelte\tsym\ttone" in lines
assert "web/StatusChip.svelte\tcss\tchip" in lines
def test_an_edit_to_an_existing_file_adds_no_file_line(tmp_path):
with http_sink(b'{"note_ids":[]}') as (port, _seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
repo = _repo(tmp_path)
(repo / "lib.py").write_text("x = 1\n")
_pre_write(env, repo, "lib.py", "def helper():\n return 1\n", tool="Edit")
lines = _ledger(tmp_path).read_text().splitlines()
assert lines == ["lib.py\tsym\thelper"]
def test_a_skipped_path_notes_nothing(tmp_path):
with http_sink(b'{"note_ids":[]}') as (port, _seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
repo = _repo(tmp_path)
_pre_write(env, repo, "README.md", "# def helper():\n")
assert not _ledger(tmp_path).exists()
# ── the Stop hook asks ────────────────────────────────────────────────────
def test_nothing_written_is_silent_and_asks_nothing(tmp_path):
with http_sink(b'{"status":"ok","reason":"x"}') as (port, seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
assert _stop(env, _repo(tmp_path)) == ""
assert seen == []
def test_all_judged_passes_silently_and_empties_the_ledger(tmp_path):
with http_sink(b'{"status":"ok","outcome":"passed","unjudged":[]}') as (port, seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
ledger = _write_ledger(tmp_path, ["a.py\tsym\thelper", "a.py\tsym\thelper"])
assert _stop(env, _repo(tmp_path)) == ""
assert seen[0]["phase"] == ["check"]
assert seen[0]["written"] == ["a.py\tsym\thelper"], "duplicates collapse"
assert seen[0]["repo"] == ["https://git.example.com/alice/widget.git"]
assert not ledger.exists()
def test_unjudged_blocks_once_in_the_servers_words_then_records_the_answer(tmp_path):
reply = json.dumps({"status": "ok", "outcome": "blocked", "reason": REASON}).encode()
with http_sink(reply) as (port, seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
repo = _repo(tmp_path)
ledger = _write_ledger(tmp_path, ["web/A.svelte\tfile\tA"])
assert json.loads(_stop(env, repo)) == {"decision": "block", "reason": REASON}
assert ledger.exists(), "kept, so the stop after the block asks about the same set"
# The stop after the block: recorded, never blocked, ledger consumed.
assert _stop(env, repo, active=True) == ""
assert [q["phase"] for q in seen] == [["check"], ["after"]]
assert not ledger.exists()
assert _stop(env, repo, active=True) == ""
assert len(seen) == 2
def test_no_block_without_a_reason_from_the_instance(tmp_path):
with http_sink(b'{"status":"ok"}') as (port, _seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
_write_ledger(tmp_path, ["a.py\tsym\thelper"])
assert _stop(env, _repo(tmp_path)) == ""
def test_another_hooks_block_loop_is_left_alone(tmp_path):
with http_sink(b'{"status":"ok","reason":"x"}') as (port, seen):
env = _env(tmp_path, f"http://127.0.0.1:{port}")
ledger = _write_ledger(tmp_path, ["a.py\tsym\thelper"])
assert _stop(env, _repo(tmp_path), active=True) == ""
assert seen == []
assert ledger.exists(), "this hook's own question is still owed"
def test_an_unreachable_instance_keeps_the_question_for_later(tmp_path):
env = _env(tmp_path) # port 9: nothing listens
ledger = _write_ledger(tmp_path, ["a.py\tsym\thelper"])
assert _stop(env, _repo(tmp_path)) == ""
assert ledger.exists()