Self-surfacing DRY — duplicate families named at the write and on arrival, tool-agnostic (milestone 299 steps 1–4) #125

Merged
bvandeusen merged 5 commits from dev into main 2026-08-22 13:41:24 -04:00
6 changed files with 295 additions and 3 deletions
Showing only changes of commit 2324c15418 - Show all commits
+16 -1
View File
@@ -258,14 +258,22 @@ fi
# the sync nudge when the recorded file itself is edited later.
state_dir="${TMPDIR:-/tmp}/scribe-priorart"
mkdir -p "$state_dir" 2>/dev/null || true
#
# A THIRD channel (#2900): the ledger's derive arm names a duplicate family
# (a derive group id) or a canon elsewhere (`canon:<snippet_id>`) for the
# shapes being written. Keyed by that token, not a note id, so it dedups on
# its own file and a family is named once per session, not at every edit.
idfile=""
syncfile=""
derivefile=""
exclude_q=""
sync_exclude_q=""
derive_exclude_q=""
if [ -n "$session_id" ]; then
safe_sid=$(printf '%s' "$session_id" | tr -c 'A-Za-z0-9._-' '_')
idfile="$state_dir/${safe_sid}.ids"
syncfile="$state_dir/${safe_sid}.sync.ids"
derivefile="$state_dir/${safe_sid}.derive.ids"
if [ -f "$idfile" ]; then
seen=$(tr '\n' ',' < "$idfile" 2>/dev/null | sed 's/,$//')
[ -n "$seen" ] && exclude_q="&exclude_ids=${seen}"
@@ -274,13 +282,17 @@ if [ -n "$session_id" ]; then
sync_seen=$(tr '\n' ',' < "$syncfile" 2>/dev/null | sed 's/,$//')
[ -n "$sync_seen" ] && sync_exclude_q="&exclude_sync_ids=${sync_seen}"
fi
if [ -f "$derivefile" ]; then
derive_seen=$(tr '\n' ',' < "$derivefile" 2>/dev/null | sed 's/,$//' | jq -sRr '@uri' 2>/dev/null) || derive_seen=""
[ -n "$derive_seen" ] && derive_exclude_q="&exclude_derive=${derive_seen}"
fi
fi
# `|| true`, not `|| exit 0`: an unreachable instance must not discard a local
# finding that needed no instance to produce.
body=$(curl -fsS --max-time 5 \
-H "Authorization: Bearer ${token}" \
"${url%/}/api/plugin/prior-art?path=${path_enc}&code=${code_enc}${repo_q}${exclude_q}${sync_exclude_q}${shapes_q}" 2>/dev/null) || body=""
"${url%/}/api/plugin/prior-art?path=${path_enc}&code=${code_enc}${repo_q}${exclude_q}${sync_exclude_q}${derive_exclude_q}${shapes_q}" 2>/dev/null) || body=""
context=""
if [ -n "$body" ]; then
@@ -295,6 +307,9 @@ if [ -n "$body" ]; then
if [ -n "$syncfile" ]; then
printf '%s' "$body" | jq -r '(.sync_note_ids // [])[]?' 2>/dev/null >> "$syncfile" || true
fi
if [ -n "$derivefile" ]; then
printf '%s' "$body" | jq -r '(.derive_keys // [])[]?' 2>/dev/null >> "$derivefile" || true
fi
fi
fi
+8
View File
@@ -129,6 +129,10 @@ async def write_path_prior_art():
surfaced. A separate channel on purpose: a reuse
hint shown early must not suppress the record-sync
nudge when the recorded file is edited later.
exclude_derive (opt) — comma-separated derive keys (a derive group id
or `canon:<snippet_id>`) already named this
session by the ledger arm (#2900); its own
channel, like the two above.
shapes (opt) — comma-separated `kind:name` definitions the hook
found in (or enclosing) the payload, kind being
css|sym. The shape ledger's write-path feed
@@ -144,6 +148,9 @@ async def write_path_prior_art():
project_id, repo, _unbound = await _project_scope()
exclude_ids = _int_list(request.args.get("exclude_ids"))
exclude_sync_ids = _int_list(request.args.get("exclude_sync_ids"))
exclude_derive = [
p.strip() for p in (request.args.get("exclude_derive") or "").split(",") if p.strip()
]
shapes = _parse_shapes(request.args.get("shapes") or "")
api_key = getattr(g, "api_key", None)
may_stamp = api_key is None or getattr(api_key, "scope", "") == "write"
@@ -153,6 +160,7 @@ async def write_path_prior_art():
exclude_ids=exclude_ids, exclude_sync_ids=exclude_sync_ids,
stamp_shapes=shapes if may_stamp else None,
repo_key=repo_bindings_svc.normalize_repo_key(repo) if repo else "",
exclude_derive=exclude_derive,
)
return jsonify(result)
+49 -2
View File
@@ -706,6 +706,7 @@ async def build_write_path_hint(
exclude_sync_ids: list[int] | None = None,
stamp_shapes: list[tuple[str, str]] | None = None,
repo_key: str = "",
exclude_derive: list[str] | None = None,
) -> dict:
"""Prior-art hint for the plugin's PreToolUse hook on Write/Edit.
@@ -765,7 +766,7 @@ async def build_write_path_hint(
"""
cfg = await get_writepath_config(user_id)
empty = {"context": "", "note_ids": [], "sync_note_ids": [], "config": cfg,
"stamped": [], "divergence": []}
"stamped": [], "divergence": [], "derive": [], "derive_keys": []}
path = (path or "").strip()
if not cfg["enabled"] or not path:
return empty
@@ -935,7 +936,20 @@ async def build_write_path_hint(
)
except Exception:
logger.warning("write-time divergence check failed", exc_info=True)
if not synced and not menu and not stamped and not divergence:
# The in-band DERIVE check (#2900): the ledger's own knowledge of the
# names being written — a duplicate family with no canon, or a canon
# recorded elsewhere. This is the arm the by-name local grep could not
# be: it knows whether the other copies are canon or stray. Keyed per
# session (`exclude_derive`) so a family is named once, not per edit.
derive: list[dict] = []
if stamp_shapes and project_id:
try:
found = await shape_ledger_svc.write_time_derive(project_id, path, stamp_shapes)
skip = set(exclude_derive or [])
derive = [d for d in found if d.get("key") not in skip]
except Exception:
logger.warning("write-time derive check failed", exc_info=True)
if not synced and not menu and not stamped and not divergence and not derive:
return empty
owners = await owner_names_for({
@@ -1003,6 +1017,8 @@ async def build_write_path_hint(
lines.append(_stamp_line(path, stamped))
if divergence:
lines.append(_divergence_line(path, divergence))
if derive:
lines.append(_derive_line(path, derive))
# Split by arm, which is the whole reason this table exists. The place arm
# carries no score and so has no home in retrieval_logs; before #2085 a
@@ -1027,9 +1043,40 @@ async def build_write_path_hint(
"config": cfg,
"stamped": stamped,
"divergence": divergence,
"derive": derive,
"derive_keys": [d["key"] for d in derive],
}
def _derive_line(path: str, derive: list[dict]) -> str:
"""The ledger's word on the names being written (#2900): a duplicate
family to derive, or a canon to reuse — said at the write."""
parts = []
for d in derive:
if d.get("canon"):
c = d["canon"]
parts.append(
f"`{c['label']}` is canon — snippet #{c['snippet_id']} at `{c['path']}`; "
"pull it and reuse, don't redefine"
)
continue
f = d["family"]
how = "identical body" if f.get("identical") else "same name defined"
files = ", ".join(f"`{x}`" for x in f.get("files") or [])
more = f.get("file_count", 0) - len(f.get("files") or [])
if more > 0:
files += f" +{more} more"
parts.append(
f"`{f['label']}` is a duplicate family with no canon — {how} in "
f"{f.get('file_count', 0)} other file(s): {files}; derive it now: "
"record the canon (create_snippet) and make the copies instances "
"(classify_shapes) — or, if these are convention not copies, "
"`classify_shapes(..., status=\"exempt\", reason_code=\"convention-plumbing\")` "
"dismisses the family — rather than adding another copy"
)
return f"> Shape ledger at `{path}`: " + "; ".join(parts) + "."
def _divergence_line(path: str, divergence: list[dict]) -> str:
"""Button B where button A is canon — named at the write (#2793)."""
parts = [
+69
View File
@@ -1594,6 +1594,75 @@ async def write_time_divergence(
return out
# How many other files a family line names before "…" — enough to go look,
# not a wall.
_DERIVE_FILES_SHOWN = 4
async def write_time_derive(
project_id: int, path: str, shapes: list[tuple[str, str]]
) -> list[dict]:
"""The in-band DERIVE check (#2900): for each (kind, name) the hook
named at ``path``, what the ledger already knows about that name
elsewhere in the project —
family the name sits in a derive-first group (identical body in N
files, or the same name in ≥3): "this is a known duplicate
family with no canon — derive it now, don't add a copy";
canon a `canonical` row of that name at another path: "this is
canon #N at <path> — reuse, don't redefine".
Only for shapes not yet judged at ``path`` (a judged shape is not
re-litigated at every edit), never for the canon's own file. Returns
[{symbol, kind, key, family?|canon?}] — `key` is the dedup token the
hook keeps per session (the group id, or canon:<snippet_id>)."""
wanted = {(k, _norm_symbol(n)): n for k, n in shapes if n}
if not wanted:
return []
async with async_session() as session:
rows = (
await session.execute(
select(CodeShape).where(
CodeShape.project_id == project_id,
CodeShape.vanished_at.is_(None),
CodeShape.symbol.in_({norm for (_k, norm) in wanted}),
)
)
).scalars().all()
out: list[dict] = []
for (kind, norm), name in wanted.items():
same = [r for r in rows if r.kind == kind and _norm_symbol(r.symbol) == norm]
here = next((r for r in same if r.path == path), None)
if here is not None and here.status not in _MECHANICAL_TODO:
continue # judged here (or this IS the canon): nothing to say
others = [r for r in same if r.path != path]
label = ("." if kind == "css" else "") + name
canon = next((r for r in others if r.status == "canonical" and r.snippet_id), None)
if canon is not None:
out.append({"symbol": name, "kind": kind, "key": f"canon:{canon.snippet_id}",
"canon": {"snippet_id": canon.snippet_id, "path": canon.path,
"label": label}})
continue
grouped = [r for r in others if r.proposal_group and r.status in _MECHANICAL_TODO]
if here is not None and here.proposal_group:
grouped = [r for r in grouped if r.proposal_group == here.proposal_group] or grouped
if not grouped:
continue
group = grouped[0].proposal_group
members = [r for r in grouped if r.proposal_group == group]
files = sorted({r.path for r in members})
out.append({
"symbol": name, "kind": kind, "key": group,
"family": {
"group": group, "label": label,
"identical": not group.startswith("name:"),
"files": files[:_DERIVE_FILES_SHOWN], "file_count": len(files),
"size": len(members) + (1 if here is not None else 0),
},
})
return out
async def flag_divergence(project_id: int, *, since: datetime | None) -> int:
"""Flag shapes created after ``since`` (the previous refresh) that sit
where a canon dominates and were not proposed as that canon. With no
+43
View File
@@ -597,6 +597,49 @@ async def test_derive_groups_land_on_rows_and_in_the_summary(seeded):
assert {r.symbol for r in rows} == {"slug"} # 2 files < the name floor
@pytest.mark.integration
async def test_write_time_derive_names_the_family_or_the_canon_for_a_name(seeded):
"""#2900: against real rows — a name in a dup family → the family (other
files, count); a name whose canonical row lives elsewhere → that canon;
a judged row at the path, the canon's own file, or an unknown name →
silence."""
from scribe.services.shape_ledger import apply_derive_groups, write_time_derive
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
defs = _defs(
("v/A.vue", "css", "log-empty", ".log-empty {", ".log-empty { color: red }"),
("v/B.vue", "css", "log-empty", ".log-empty {", ".log-empty { color: red }"),
("v/C.vue", "css", "log-empty", ".log-empty {", ".log-empty { color: red }"),
("src/factory.py", "sym", "factory", "def factory():", "def factory():\n return 1"),
)
await sync_repo_shapes(pid, REPO, defs, seen_marker="m1")
await classify_shapes(owner, pid, [
{"path": "src/factory.py", "symbol": "factory", "status": "canonical", "snippet_id": sid},
])
assert await apply_derive_groups(pid) >= 3
# A 4th copy about to be written → the family, naming the other files.
out = await write_time_derive(pid, "v/D.vue", [("css", "log-empty"), ("css", "unknown")])
assert len(out) == 1 and out[0]["symbol"] == "log-empty" and out[0]["kind"] == "css"
fam = out[0]["family"]
assert fam["identical"] is True and fam["label"] == ".log-empty"
assert fam["files"] == ["v/A.vue", "v/B.vue", "v/C.vue"] and fam["file_count"] == 3
assert out[0]["key"] == fam["group"] and fam["group"].startswith("dup:")
# Editing one existing member still names the OTHER members.
out = await write_time_derive(pid, "v/A.vue", [("css", "log-empty")])
assert out[0]["family"]["files"] == ["v/B.vue", "v/C.vue"] and out[0]["family"]["size"] == 3
# The canon's name elsewhere → the canon; in the canon's own file → silence.
out = await write_time_derive(pid, "src/other.py", [("sym", "factory")])
assert out == [{"symbol": "factory", "kind": "sym", "key": f"canon:{sid}",
"canon": {"snippet_id": sid, "path": "src/factory.py", "label": "factory"}}]
assert await write_time_derive(pid, "src/factory.py", [("sym", "factory")]) == []
# A judged row at the path is not re-litigated.
await classify_shapes(owner, pid, [
{"path": "v/B.vue", "symbol": "log-empty", "status": "exempt", "reason": "print sheet"},
])
assert await write_time_derive(pid, "v/B.vue", [("css", "log-empty")]) == []
@pytest.mark.integration
async def test_derive_new_names_the_copy_that_joined_a_family_since_the_stamp(seeded):
"""#2899: the first sync seeds one `slug`; a later sync adds an identical
+110
View File
@@ -1243,6 +1243,116 @@ def test_hook_sends_the_enclosing_definition_for_a_body_edit(tmp_path):
assert seen["shapes"] == ["sym:onTrash"]
@pytest.mark.asyncio
async def test_the_write_time_derive_check_names_a_family_or_a_canon_in_band():
"""#2900: the ledger's own word on the names being written — a duplicate
family with no canon, or a canon recorded elsewhere — rendered at the
write even when nothing else does; keyed so the session's exclude
channel silences a family already named."""
from scribe.services import plugin_context as pc
found = [
{"symbol": "log-empty", "kind": "css", "key": "dup:483a",
"family": {"group": "dup:483a", "label": ".log-empty", "identical": True,
"files": ["a/TaskLogSection.vue", "a/WorkspaceTaskPanel.vue"],
"file_count": 5, "size": 6}},
{"symbol": "btn-primary", "kind": "css", "key": "canon:2855",
"canon": {"snippet_id": 2855, "path": "frontend/src/assets/components.css",
"label": ".btn-primary"}},
{"symbol": "load", "kind": "sym", "key": "name:sym:load",
"family": {"group": "name:sym:load", "label": "load", "identical": False,
"files": ["a/X.vue", "a/Y.vue", "a/Z.vue"], "file_count": 3, "size": 4}},
]
check = AsyncMock(return_value=found)
patches = dict(
get_writepath_config=AsyncMock(return_value=_cfg()),
semantic_search_notes=AsyncMock(return_value=[]),
record_retrieval=MagicMock(), owner_names_for=AsyncMock(return_value={}),
)
with patch.multiple(pc, **patches), \
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
patch.object(pc.shape_ledger_svc, "recent_pulls", AsyncMock(return_value={})), \
patch.object(pc.shape_ledger_svc, "write_time_divergence", AsyncMock(return_value=[])), \
patch.object(pc.shape_ledger_svc, "write_time_derive", check):
out = await pc.build_write_path_hint(
1, "frontend/src/components/New.vue", code=REAL_CODE, project_id=24,
stamp_shapes=[("css", "log-empty"), ("css", "btn-primary"), ("sym", "load")],
exclude_derive=["name:sym:load"],
)
check.assert_awaited_once_with(24, "frontend/src/components/New.vue",
[("css", "log-empty"), ("css", "btn-primary"), ("sym", "load")])
# The excluded family is gone; the other two render and are keyed.
assert [d["key"] for d in out["derive"]] == ["dup:483a", "canon:2855"]
assert out["derive_keys"] == ["dup:483a", "canon:2855"]
ctx = out["context"]
assert "Shape ledger at `frontend/src/components/New.vue`" in ctx
assert "`.log-empty` is a duplicate family with no canon — identical body in 5 other file(s): " \
"`a/TaskLogSection.vue`, `a/WorkspaceTaskPanel.vue` +3 more; derive it now" in ctx
assert "`.btn-primary` is canon — snippet #2855 at `frontend/src/assets/components.css`" in ctx
assert "convention-plumbing" in ctx
assert "`load`" not in ctx
# No project → no check; a failing check never sinks the hint.
check.reset_mock()
with patch.multiple(pc, **patches), \
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
patch.object(pc.shape_ledger_svc, "recent_pulls", AsyncMock(return_value={})), \
patch.object(pc.shape_ledger_svc, "write_time_divergence", AsyncMock(return_value=[])), \
patch.object(pc.shape_ledger_svc, "write_time_derive", check):
out = await pc.build_write_path_hint(1, "x.py", code=REAL_CODE, stamp_shapes=[("sym", "f")])
check.assert_not_awaited()
assert out["derive"] == [] and out["derive_keys"] == []
boom = AsyncMock(side_effect=RuntimeError("ledger down"))
with patch.multiple(pc, **patches), \
patch.object(pc.snippets_svc, "list_snippets", AsyncMock(return_value=([], 0))), \
patch.object(pc.shape_ledger_svc, "recent_pulls", AsyncMock(return_value={})), \
patch.object(pc.shape_ledger_svc, "write_time_divergence", AsyncMock(return_value=[])), \
patch.object(pc.shape_ledger_svc, "write_time_derive", boom):
out = await pc.build_write_path_hint(1, "x.py", code=REAL_CODE, project_id=24,
stamp_shapes=[("sym", "f")])
assert out["context"] == "" and out["derive"] == []
def test_the_hook_keeps_a_derive_channel_and_sends_it_back(tmp_path):
"""#2900: derive keys the server returns land in the session's own
`.derive.ids` file and go back as `exclude_derive` on the next write —
a family is named once per session, not at every edit."""
import http.server
import threading
import urllib.parse
seen: list[dict] = []
class _Sink(http.server.BaseHTTPRequestHandler):
def do_GET(self):
seen.append(urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query))
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(b'{"context":"> family","note_ids":[],"sync_note_ids":[],'
b'"derive_keys":["dup:483a","canon:2855"]}')
def log_message(self, *a):
pass
server = http.server.HTTPServer(("127.0.0.1", 0), _Sink)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
env = dict(_hook_runtime_env(), SCRIBE_URL=f"http://127.0.0.1:{server.server_port}",
TMPDIR=str(tmp_path))
payload = {"session_id": "s-derive-1", "cwd": str(tmp_path), "tool_name": "Write",
"tool_input": {"file_path": str(tmp_path / "a.css"),
"content": ".log-empty {\n color: red;\n}\n"}}
for _ in range(2):
out = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload),
capture_output=True, text=True, env=env)
assert out.returncode == 0, out.stderr
finally:
server.shutdown()
assert "exclude_derive" not in seen[0]
assert seen[1]["exclude_derive"] == ["dup:483a,canon:2855"]
state = tmp_path / "scribe-priorart" / "s-derive-1.derive.ids"
assert state.read_text().split() == ["dup:483a", "canon:2855", "dup:483a", "canon:2855"]
@pytest.mark.asyncio
async def test_the_write_time_divergence_check_is_named_in_band():
"""#2793: the hook named a shape at a path whose directory a canon