From cdb71e64c4bf45a95e326ba15aa50a461de2111f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 26 Oct 2026 17:41:21 -0400 Subject: [PATCH] fix(6477): a body uses a canon by name only when the name can resolve to it The proposer's reference basis, and every uses edge it writes, counted any word-bounded mention of a canon's symbol. Three holes made that wrong at scale (snippet #5448, symbol `file`, held 2,161 uses edges): - Go, Rust, Kotlin, Swift, Dart, C, C#, Ruby and PHP were outside the language-family maps, so their canons read as "unknown, no gate" and were matched in every Python, TS and CSS body. - Canons in other projects were matched by name, though one repo never calls a function defined in another's. - A plain word (`file`, `seen`) or a module's own helper (`_run`) matched wherever it appeared. names_canon is now the one by-name test, used by match_canon, reference_canons and the write-path hint: same project, not a name the file defines itself (unless the canon is recorded there), no generic verb with or without a leading underscore, and a bare lowercase word only where it is called. The write path keeps the payload's own names in play, since a redefinition of a pulled canon is the copy it exists to flag. Repair: record_uses only adds, so replace_reference_uses now makes a row's machine-written reference edges exactly what the current rule finds, leaving judged and hook edges alone. _PROPOSER_VERSION 7 re-reads every row on its project's next refresh. Migration 0125 deletes the cross-project reference edges already stored. Co-Authored-By: Claude Opus 5.5 --- .../0125_drop_cross_project_reference_uses.py | 35 +++++ src/grimoire/services/shape_ledger.py | 133 +++++++++++++++--- tests/test_integration_shape_classify.py | 37 +++++ tests/test_shape_ledger.py | 47 +++++++ 4 files changed, 229 insertions(+), 23 deletions(-) create mode 100644 alembic/versions/0125_drop_cross_project_reference_uses.py diff --git a/alembic/versions/0125_drop_cross_project_reference_uses.py b/alembic/versions/0125_drop_cross_project_reference_uses.py new file mode 100644 index 00000000..a12cf63d --- /dev/null +++ b/alembic/versions/0125_drop_cross_project_reference_uses.py @@ -0,0 +1,35 @@ +"""drop machine-written uses edges that cross projects + +Revision ID: 0125 +Revises: 0124 +Create Date: 2026-10-26 + +A `uses` edge with basis `reference` is the proposer's by-name reading: the +shape's body names a canon's symbol. A body never calls a function defined in +another project's repository, so such an edge across projects is a common +word matched, not a call. The proposer no longer writes them, and this removes +the ones it wrote before. Edges a person or an audit judged keep their own +basis and are left alone. Rows within one project are re-read on the next +coverage refresh, which replaces their reference edges under the same rule. + +Downgrade restores nothing: the removed edges were never facts. +""" +from alembic import op + +revision = "0125" +down_revision = "0124" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + "DELETE FROM code_shape_uses u USING code_shapes s, notes n " + "WHERE u.shape_id = s.id AND u.snippet_id = n.id " + "AND u.basis = 'reference' " + "AND n.project_id IS NOT NULL AND n.project_id <> s.project_id" + ) + + +def downgrade() -> None: + pass diff --git a/src/grimoire/services/shape_ledger.py b/src/grimoire/services/shape_ledger.py index 475470c0..db17e4f8 100644 --- a/src/grimoire/services/shape_ledger.py +++ b/src/grimoire/services/shape_ledger.py @@ -320,6 +320,31 @@ async def record_uses( return n +async def replace_reference_uses( + session, row: CodeShape, snippet_ids, *, evidence: str | None = None, +) -> int: + """Make the row's mechanical `reference` edges exactly ``snippet_ids``. + + `record_uses` only adds, so a by-name rule that tightens would otherwise + leave every edge the looser rule wrote. Only `reference` edges are + dropped: a judgment, or the write-path hook's evidence, outlives a + re-read of the body. Returns the number of edges written or refreshed.""" + wanted = {int(x) for x in (snippet_ids or []) if x} + if row.id is not None: + stale = ( + await session.execute( + select(CodeShapeUse).where( + CodeShapeUse.shape_id == row.id, + CodeShapeUse.basis == "reference", + ) + ) + ).scalars().all() + for edge in stale: + if edge.snippet_id not in wanted: + await session.delete(edge) + return await record_uses(session, row, wanted, basis="reference", evidence=evidence) + + async def uses_of(shape_ids) -> dict[int, list[CodeShapeUse]]: """{shape_id: [edges]} for a set of rows — the read side of record_uses.""" ids = [int(x) for x in shape_ids if x] @@ -1553,8 +1578,19 @@ async def suggest_write_path_instances( continue fields = snippet_fields(note) symbol = fields.get("symbol") or "" - kind = snippet_kind(symbol, fields.get("language") or "") - if references_symbol(code, symbol, kind): + language = (fields.get("language") or "").strip().lower() + kind = snippet_kind(symbol, language) + canon = Canon( + int(sid), kind, symbol, + tuple(((loc.get("path") or ""), (loc.get("symbol") or "")) + for loc in fields.get("locations") or []), + "", "", int(note.project_id or 0), language, + ) + # A redefinition of the pulled name in this payload is the copy the + # hint exists to catch, so the payload's own names are not "local". + if (kind != "sym" or same_family(path, language)) and names_canon( + kind, path, code, canon, project_id=project_id, + ): rank, why = 2, _REFERENCE_REASON.format( sid=sid, symbol=_norm_symbol(symbol) ) @@ -1762,7 +1798,7 @@ def _semantic_priority(row) -> tuple: # were examined under, so a tightened rule re-examines everything once. A # change that does not alter what is proposed needs no bump. A stale "miss" # basis on an old row is inert: a miss is not evidence (see _SEMANTIC_LIMIT). -_PROPOSER_VERSION = 6 +_PROPOSER_VERSION = 7 # Signature resemblance floor, name blanked (difflib ratio) — and a length # floor, because `def NAME():` resembles `def NAME(x):` at 0.95 while saying # nothing; a family shape has parameters to resemble. @@ -1805,6 +1841,13 @@ _FAMILY_BY_LANG = { "css": "css", "scss": "css", "sass": "css", "less": "css", "bash": "sh", "sh": "sh", "shell": "sh", "zsh": "sh", "sql": "sql", + "go": "go", "golang": "go", + "rust": "rs", "rs": "rs", + "java": "jvm", "kotlin": "jvm", "kt": "jvm", + "swift": "swift", "dart": "dart", + "c": "c", "cpp": "c", "c++": "c", "h": "c", + "csharp": "cs", "c#": "cs", "cs": "cs", + "ruby": "rb", "rb": "rb", "php": "php", } _FAMILY_BY_EXT = { ".py": "py", ".pyi": "py", @@ -1812,6 +1855,11 @@ _FAMILY_BY_EXT = { ".css": "css", ".scss": "css", ".sass": "css", ".less": "css", ".sh": "sh", ".bash": "sh", ".zsh": "sh", ".sql": "sql", + ".go": "go", ".rs": "rs", + ".java": "jvm", ".kt": "jvm", ".kts": "jvm", + ".swift": "swift", ".dart": "dart", + ".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c", + ".cs": "cs", ".rb": "rb", ".php": "php", } @@ -1877,11 +1925,48 @@ def text_contains(body: str, code: str) -> bool: return a in b or b in a -def reference_canons(kind: str, path: str, symbol: str, body: str, canons: Iterable[Canon]) -> list[int]: +def names_canon( + kind: str, path: str, body: str, canon: Canon, *, + project_id: int = 0, local_names: frozenset[str] = frozenset(), +) -> bool: + """Does this body name THIS canon, rather than merely contain its word? + + The by-name test behind the reference basis and every uses edge the + proposer writes. A mention counts only when it can resolve to the canon: + - never across projects: one repo does not call a function defined in + another's, however alike the names are; + - never a name the file defines itself (a module's own `_run` helper), + unless the canon is recorded in this very file; + - never a generic verb, with or without a leading underscore; + - a bare lowercase word (`file`, `seen`, `seal`) only where it is called + as `word(`, because as a plain word it is everyone's variable. + Same kind and language family are checked by the callers' loops.""" + sym = _norm_symbol(canon.symbol or "") + if not sym: + return False + if project_id and canon.project_id and canon.project_id != project_id: + return False + if sym.lstrip("_").lower() in _REFERENCE_STOPLIST: + return False + if sym in local_names: + from grimoire.services.snippets import _path_touches + + if not any(lp and _path_touches(lp, path) for lp, _ls in canon.locations): + return False + if kind == "sym" and re.fullmatch(r"[a-z]+", sym): + return re.search(rf"(? list[int]: """Every canon this body NAMES — the uses edges the proposer can - write mechanically: same kind, same language family, symbol not in the - generic-verb stoplist, and not the shape's own name.""" + write mechanically: same kind, same language family, not the shape's own + name, and named in the sense `names_canon` gives.""" norm_sym = _norm_symbol(symbol) + local = frozenset(local_names) out: list[int] = [] for c in canons: if c.kind != kind or not c.symbol: @@ -1890,9 +1975,7 @@ def reference_canons(kind: str, path: str, symbol: str, body: str, canons: Itera continue if _norm_symbol(c.symbol) == norm_sym: continue - if _norm_symbol(c.symbol).lower() in _REFERENCE_STOPLIST: - continue - if references_symbol(body, c.symbol, kind): + if names_canon(kind, path, body, c, project_id=project_id, local_names=local): out.append(c.snippet_id) return out @@ -1900,6 +1983,7 @@ def reference_canons(kind: str, path: str, symbol: str, body: str, canons: Itera def match_canon( kind: str, path: str, symbol: str, signature: str, body: str, canons: Iterable[Canon], *, project_id: int = 0, + local_names: frozenset[str] = frozenset(), ) -> tuple[int, str, float] | None: """The strongest (snippet_id, basis, score) a shape earns against the canon catalog, by the non-semantic bases — or None. Strongest by @@ -1924,11 +2008,7 @@ def match_canon( if not any(location_covers(lp, ls, path, symbol) for lp, ls in c.locations): offer("symbol", 1.0, c) continue # its own location is canonical territory, not a proposal - if ( - c.symbol - and _norm_symbol(c.symbol).lower() not in _REFERENCE_STOPLIST - and references_symbol(body, c.symbol, kind) - ): + if names_canon(kind, path, body, c, project_id=project_id, local_names=local_names): offer("reference", 0.9, c) if c.code_norm and text_contains(body, c.code_norm): offer("text", 0.95, c) @@ -2064,6 +2144,12 @@ async def propose_for_repo( if canons is None: canons = await canon_catalog(user_id) by_key = {(d[0], d[1], d[2]): d for d in definitions} + # What each file defines itself: a name it defines resolves there. + names_by_path: dict[str, set[str]] = {} + for d in definitions: + if d[1] == "sym": + names_by_path.setdefault(d[0], set()).add(_norm_symbol(d[2])) + defined_in = {p: frozenset(n) for p, n in names_by_path.items()} # The semantic arm is the widest net and, across projects, is noise: it # is held to the shape's own project and language family. The precise # bases (symbol/text) still reach family canon in other projects. @@ -2102,15 +2188,16 @@ async def propose_for_repo( examined_as = f"{body_sha}@{_PROPOSER_VERSION}" if row.proposed_at is not None and row.proposed_sha == examined_as: continue + local = defined_in.get(row.path, frozenset()) if row.status not in _MECHANICAL_TODO: # A judged row gets no proposal — but its uses edges # are a fact about the body, judged or not: the consumer map # of a canon must include the call sites someone already # classified. Mark it examined so the scan runs once per body. - used = reference_canons(row.kind, row.path, row.symbol, body, canons) - if used: - await record_uses(session, row, used, basis="reference", - evidence="proposer: body names the canon's symbol") + used = reference_canons(row.kind, row.path, row.symbol, body, canons, + project_id=project_id, local_names=local) + await replace_reference_uses(session, row, used, + evidence="proposer: body names the canon's symbol") row.proposed_at = now row.proposed_sha = examined_as continue @@ -2118,7 +2205,7 @@ async def propose_for_repo( group = row.proposal_group # derive grouping is reassigned below hit = match_canon( row.kind, row.path, row.symbol, signature, body, canons, - project_id=project_id, + project_id=project_id, local_names=local, ) _clear_proposal(row) row.proposal_group = group @@ -2126,10 +2213,10 @@ async def propose_for_repo( row.proposed_sha = examined_as # Consumption is recorded for every canon the body names, # whatever the row is then judged to be. - used = reference_canons(row.kind, row.path, row.symbol, body, canons) - if used: - await record_uses(session, row, used, basis="reference", - evidence="proposer: body names the canon's symbol") + used = reference_canons(row.kind, row.path, row.symbol, body, canons, + project_id=project_id, local_names=local) + await replace_reference_uses(session, row, used, + evidence="proposer: body names the canon's symbol") if hit: row.proposed_snippet_id, row.proposal_basis, row.proposal_score = hit row.proposal_group = None diff --git a/tests/test_integration_shape_classify.py b/tests/test_integration_shape_classify.py index db5a4260..5eabd88b 100644 --- a/tests/test_integration_shape_classify.py +++ b/tests/test_integration_shape_classify.py @@ -294,6 +294,43 @@ async def test_get_snippet_carries_the_structured_consumer_map(seeded): assert consumers == {"instances": [], "variants": [], "uses": []} +@pytest.mark.integration +async def test_a_reread_takes_back_the_reference_edges_a_looser_rule_wrote(seeded): + """record_uses only adds; a tightened by-name rule must also remove what + the old rule wrote. A judged edge on the same row survives.""" + from grimoire.services import snippets as snippets_svc + from grimoire.services.shape_ledger import record_uses, replace_reference_uses + + owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"] + helper = await snippets_svc.create_snippet( + owner, name="cls_reread_helper", code="def hash_token(raw):\n return raw\n", + language="python", repo="Widget", path="src/hash.py", symbol="hash_token", + project_id=pid, + ) + hid = int(helper.id) + + async def make_app_row(s): + return (await s.execute(select(CodeShape).where( + CodeShape.project_id == pid, CodeShape.symbol == "make_app", + ))).scalar_one() + + async with async_session() as s: + await record_uses(s, await make_app_row(s), [sid, hid], basis="reference") + await s.commit() + await classify_shapes(owner, pid, [ + {"path": "src/app.py", "symbol": "make_app", "status": "exempt", + "reason": "its own thing", "uses": [hid]}, + ], via="audit") + + async with async_session() as s: + await replace_reference_uses(s, await make_app_row(s), []) + await s.commit() + _, n_sid = await list_project_shapes(owner, pid, uses=sid) + rows_hid, n_hid = await list_project_shapes(owner, pid, uses=hid) + assert n_sid == 0 + assert n_hid == 1 and rows_hid[0].symbol == "make_app" + + @pytest.mark.integration async def test_sync_refiles_rows_whose_snippet_was_purged(seeded): """The SET NULL companion: a judgment whose target is hard-deleted diff --git a/tests/test_shape_ledger.py b/tests/test_shape_ledger.py index a4a38aab..31e2c0cb 100644 --- a/tests/test_shape_ledger.py +++ b/tests/test_shape_ledger.py @@ -560,6 +560,53 @@ def test_reference_canons_names_every_used_canon_not_just_the_best(): assert reference_canons("sym", "f/v.vue", "show", "fmtDate(x); hash_token(y)", [a, ts]) == [3] +def test_a_go_canon_says_nothing_about_a_python_or_vue_body(): + """Go was outside the family maps, so it read as "unknown, no gate" and a + Go whole-file canon whose symbol is `file` was matched in every body.""" + from grimoire.services.shape_ledger import Canon, _norm_text, language_family, match_canon, reference_canons + go = Canon(5, "sym", "Ledger", (("internal/files/nextcloud.go", "Ledger"),), + "type Ledger struct {", _norm_text("x"), 0, "go") + assert language_family("go") == "go" + body = "def sync(path):\n return Ledger(path)\n" + assert reference_canons("sym", "tests/test_x.py", "sync", body, [go]) == [] + assert match_canon("sym", "a.vue", "x", "function x() {", "new Ledger()", [go]) is None + assert reference_canons("sym", "internal/sync/run.go", "Sync", "l := Ledger(p)", [go]) == [5] + + +def test_a_canon_in_another_project_is_never_used_by_name(): + from grimoire.services.shape_ledger import Canon, _norm_text, match_canon, reference_canons + steward = Canon(6, "sym", "run_control", (("steward/ctl.py", "run_control"),), + "def run_control(cmd):", _norm_text("x"), 30, "python") + body = "def go():\n run_control(['ls'])\n" + assert reference_canons("sym", "src/a.py", "go", body, [steward], project_id=2) == [] + assert match_canon("sym", "src/a.py", "go", "def go():", body, [steward], project_id=2) is None + assert reference_canons("sym", "src/a.py", "go", body, [steward], project_id=30) == [6] + + +def test_a_name_the_file_defines_itself_resolves_there(): + """A test module's own `run_helper` is not a call of another file's + canon of that name; in the canon's own file it is.""" + from grimoire.services.shape_ledger import Canon, _norm_text, reference_canons + canon = Canon(7, "sym", "run_helper", (("src/tools.py", "run_helper"),), + "def run_helper(x):", _norm_text("x"), 2, "python") + body = "def test_a():\n run_helper(1)\n" + local = frozenset({"run_helper", "test_a"}) + assert reference_canons("sym", "tests/test_a.py", "test_a", body, [canon], local_names=local) == [] + assert reference_canons("sym", "src/tools.py", "caller", body, [canon], local_names=local) == [7] + assert reference_canons("sym", "tests/test_b.py", "test_a", body, [canon]) == [7] + + +def test_a_plain_word_counts_only_where_it_is_called(): + """`file`, `seen`: everyone's variable. `seal(...)` is still a call.""" + from grimoire.services.shape_ledger import Canon, _norm_text, reference_canons + seen = Canon(8, "sym", "seen", (("src/tree.py", "seen"),), "def seen(f):", _norm_text("x"), 2, "python") + assert reference_canons("sym", "src/a.py", "walk", "if path in seen:\n return", [seen]) == [] + assert reference_canons("sym", "src/a.py", "walk", "if seen (f):\n return", [seen]) == [8] + # A leading underscore does not get a generic verb past the stoplist. + run = Canon(9, "sym", "_run", (("src/x.py", "_run"),), "def _run(cmd):", _norm_text("x"), 2, "python") + assert reference_canons("sym", "src/a.py", "go", "_run(['ls'])", [run]) == [] + + def test_reason_codes_are_a_fixed_catalogue_and_validated(): """An optional index beside the prose reason; unknown codes are a structural error (the batch applies nothing)."""