diff --git a/alembic/versions/0080_code_shape_proposals.py b/alembic/versions/0080_code_shape_proposals.py new file mode 100644 index 0000000..7a6e932 --- /dev/null +++ b/alembic/versions/0080_code_shape_proposals.py @@ -0,0 +1,54 @@ +"""Shape fingerprints + the mechanical proposer's columns (#2792, milestone 294) + +Revision ID: 0080 +Revises: 0079 +Create Date: 2026-08-21 + +Two additions to the ledger. `signature` / `body_sha` fingerprint each shape +(definition line + a whitespace/comment-insensitive hash of its block) so the +proposer can match on content and a later drift recheck can notice change, +without the ledger ever storing code. The proposal columns carry the +proposer's standing suggestion for an unclassified row — instance-of-#N with +a basis and score, or a derive-first group key — and `proposed_sha` +remembers the content it was judged at so a refresh re-examines only what +changed. Mechanical and recomputable: a restore that lacks them loses +nothing the next refresh does not rebuild. +""" +import sqlalchemy as sa +from alembic import op + +revision = "0080" +down_revision = "0079" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("code_shapes", sa.Column("signature", sa.Text(), nullable=False, server_default="")) + op.add_column("code_shapes", sa.Column("body_sha", sa.Text(), nullable=False, server_default="")) + op.add_column( + "code_shapes", + sa.Column( + "proposed_snippet_id", + sa.BigInteger(), + sa.ForeignKey("notes.id", ondelete="SET NULL"), + nullable=True, + ), + ) + op.add_column("code_shapes", sa.Column("proposal_basis", sa.Text(), nullable=True)) + op.add_column("code_shapes", sa.Column("proposal_score", sa.Float(), nullable=True)) + op.add_column("code_shapes", sa.Column("proposal_group", sa.Text(), nullable=True)) + op.add_column("code_shapes", sa.Column("proposed_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("code_shapes", sa.Column("proposed_sha", sa.Text(), nullable=False, server_default="")) + op.create_index( + "ix_code_shapes_proposed", "code_shapes", ["project_id", "proposed_snippet_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_code_shapes_proposed", table_name="code_shapes") + for col in ( + "proposed_sha", "proposed_at", "proposal_group", "proposal_score", + "proposal_basis", "proposed_snippet_id", "body_sha", "signature", + ): + op.drop_column("code_shapes", col) diff --git a/frontend/src/views/ProjectView.vue b/frontend/src/views/ProjectView.vue index 906ab7e..1e54742 100644 --- a/frontend/src/views/ProjectView.vue +++ b/frontend/src/views/ProjectView.vue @@ -427,6 +427,13 @@ interface CoverageGap { unclassified: number; total: number; } +interface DeriveGroup { + group: string; + kind: string; + label: string; + size: number; + paths: string[]; +} interface Coverage { total: number; accounted: number; @@ -436,6 +443,10 @@ interface Coverage { computed_at: string; repos: { repo: string; ref: string; total: number; accounted: number }[]; largest_gaps: CoverageGap[]; + // The mechanical proposer's standing (#2792): canon proposals awaiting an + // agent's confirm, and the biggest repeats-with-no-canon families. + proposed?: number; + derive_groups?: DeriveGroup[]; } const coverage = ref(null); @@ -746,6 +757,24 @@ async function confirmDelete() { {{ gap.dir }} {{ gap.unclassified }} +
+ Proposed: + + {{ coverage.proposed }} awaiting confirm + + + {{ g.label }} ×{{ g.size }} + +

Not measured yet — Refresh reads the bound repo's definitions into diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 5fec290..f01ae03 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "scribe", "description": "Scribe system-of-record for Claude Code: MCP tools over your notes/tasks/projects/rules, a session-start push channel that surfaces your always-on rules + active-project context, process-skills (writing-plans, systematic-debugging, verification, brainstorming, reusing-code), and your saved Scribe Processes auto-surfaced as skills (/scribe:sync). Replaces superpowers + file-memory with one app-backed plugin.", - "version": "0.1.34", + "version": "0.1.35", "author": { "name": "Bryan Van Deusen" }, "mcpServers": { "scribe": { diff --git a/plugin/skills/shape-accounting/SKILL.md b/plugin/skills/shape-accounting/SKILL.md index dc4e57a..6bdbfe1 100644 --- a/plugin/skills/shape-accounting/SKILL.md +++ b/plugin/skills/shape-accounting/SKILL.md @@ -51,6 +51,27 @@ need a hand judgment: judgment: it never overrides a classification you made, and a `classify_shapes` call overrides it. +## The machine proposes, judgment classifies + +Every coverage refresh runs the **mechanical proposer** over the unclassified +rows: same symbol as a canon elsewhere → textual containment → body +references a canon → signature resemblance → semantic (capped per refresh). A hit +is a *proposal* on the row, never a classification. Work the queue in bulk: + +1. `list_shapes(project_id, proposal="canon", snippet_id=N)` or + `path="dir"` — read the page; `proposal` carries snippet_id, basis, score. +2. `confirm_shape_proposals(project_id, snippet_id=N)` (or `path=`, + `basis=`) for the ones that hold — hundreds at a time; `symbol` and + `reference` proposals are near-certain, `semantic` deserves a look. +3. `classify_shapes` the rest — variant, exempt, or instance of a different + snippet. Any judgment retires the proposal. + +`list_shapes(project_id, proposal="derive")` lists the **derive-first +candidates** — the same body in ≥2 places or the same name defined in ≥3 +files, with no canon at all (`proposal.group` names the family; the coverage +payload's `derive_groups` ranks the biggest). That is the consolidation +queue, not a classification queue: see below. + ## The derive-first rule N same-shaped occurrences matching **no** recorded canon is never N loose diff --git a/src/scribe/mcp/tools/shapes.py b/src/scribe/mcp/tools/shapes.py index 0f3485e..f86d04b 100644 --- a/src/scribe/mcp/tools/shapes.py +++ b/src/scribe/mcp/tools/shapes.py @@ -62,6 +62,7 @@ async def list_shapes( include_vanished: bool = False, limit: int = 100, offset: int = 0, + proposal: str = "", ) -> dict: """Read a project's shape ledger — `status="unclassified"` IS the todo. @@ -75,6 +76,11 @@ async def list_shapes( snippet_id: rows classified against this snippet — a consumer map. include_vanished: include shapes no longer in the tree (history). limit/offset: page through big ledgers (limit caps at 500). + proposal: the proposer's queue (#2792) — "any", "canon" (rows the + machine thinks are an instance of a snippet: `proposal` carries + snippet_id, basis, score), "derive" (rows that repeat with NO + canon: `proposal.group` names the family), or one basis + (symbol/text/reference/signature/semantic). Returns {"shapes": [...], "total": N} — total counts every match, not just this page. Each row's `classified_by` says who judged: agent / @@ -84,20 +90,61 @@ async def list_shapes( was stamped an instance with the evidence in `reason`. A hook row is overridable by any classify_shapes call; it never overrides yours. - Classify what you can judge with classify_shapes; a repeating shape - with NO recorded canon is a derive-one-first moment (consolidate onto a - reference, create_snippet it, then classify the rest against it), never - N loose classifications. + THE FAST PATH through a big todo is the proposer's queue: every coverage + refresh matches unclassified shapes against canon (strongest basis + first: same symbol elsewhere → textual containment → body references + the canon → signature resemblance → semantic) and attaches a + `proposal` to each row it can speak for. Review `proposal="canon"` by + snippet or directory, then confirm_shape_proposals the ones that hold — + hundreds at a time — and classify_shapes the rest (variant/exempt, or + instance of a different snippet). `proposal="derive"` lists the + derive-first candidates: a repeating shape with NO recorded canon is + never N loose classifications — consolidate onto a reference, + create_snippet it, then classify the group against it. """ uid = current_user_id() rows, total = await shape_ledger_svc.list_project_shapes( uid, project_id, status=status, path=path, snippet_id=snippet_id, include_vanished=include_vanished, limit=limit, offset=offset, + proposal=proposal, ) return {"shapes": [r.to_dict() for r in rows], "total": total} +async def confirm_shape_proposals( + project_id: int, + snippet_id: int = 0, + path: str = "", + basis: str = "", + min_score: float = 0.0, +) -> dict: + """Confirm the proposer's canon proposals you have reviewed, in batch. + + The machine proposes, judgment classifies (#2792): each matching row — + live, unclassified, carrying a `proposal` with a snippet_id — becomes + `instance` of that snippet, classified_by="agent", reason naming the + basis and score. Narrow to what you actually looked at: at least one of + snippet_id (confirm one canon's whole queue after reading its + `list_shapes(proposal="canon", ...)` page), path (a directory you + audited), or basis (e.g. "symbol" and "reference" are near-certain; + "semantic" deserves a look first) is required — a bare confirm-all is + not a judgment. min_score trims a basis's tail. + + Proposals you do NOT confirm are judged with classify_shapes (variant, + exempt, or instance of a different snippet) — any judgment retires the + proposal. Requires write access. Returns {"confirmed": N}. + """ + uid = current_user_id() + try: + return await shape_ledger_svc.confirm_proposals( + uid, project_id, snippet_id=snippet_id, path=path, basis=basis, + min_score=min_score, + ) + except ValueError as exc: + return {"error": str(exc)} + + async def refresh_pattern_coverage(project_id: int) -> dict: """Seed or refresh the project's shape ledger NOW, and return the readout. @@ -114,9 +161,17 @@ async def refresh_pattern_coverage(project_id: int) -> dict: owner adds one (Settings → Integrations → Git Forges); no served repo → bind_repo on a host a connection serves. + The refresh is also when the mechanical proposer runs (#2792): with the + repo bodies in hand it matches every changed unclassified shape against + canon and records proposals (see list_shapes proposal=), then regroups + the derive-first candidates. Semantic matching is capped per refresh, so + a large ledger's queue grows across refreshes rather than in one. + Returns the accounting payload — total, accounted, counts by status, - unclassified, repos, largest_gaps — plus `pattern_coverage`, the same - one-line summary enter_project carries. + unclassified, repos, largest_gaps, `proposed` (canon proposals awaiting + confirmation), `derive_groups` (the biggest repeats-with-no-canon + families), `proposer` (what this refresh examined) — plus + `pattern_coverage`, the same one-line summary enter_project carries. """ uid = current_user_id() coverage = await coverage_svc.refresh_for_caller(uid, project_id) @@ -127,5 +182,8 @@ async def refresh_pattern_coverage(project_id: int) -> dict: def register(mcp) -> None: - for fn in (classify_shapes, list_shapes, refresh_pattern_coverage): + for fn in ( + classify_shapes, list_shapes, refresh_pattern_coverage, + confirm_shape_proposals, + ): mcp.tool(name=fn.__name__)(fn) diff --git a/src/scribe/models/code_shape.py b/src/scribe/models/code_shape.py index 7999618..6775d5b 100644 --- a/src/scribe/models/code_shape.py +++ b/src/scribe/models/code_shape.py @@ -3,6 +3,7 @@ from datetime import datetime from sqlalchemy import ( BigInteger, DateTime, + Float, ForeignKey, Index, Integer, @@ -19,6 +20,12 @@ from scribe.models.base import TimestampMixin SHAPE_STATUSES = ("canonical", "instance", "variant", "exempt", "unclassified") SHAPE_CLASSIFIERS = ("agent", "audit", "hook", "mechanical", "import") +# How the mechanical proposer (#2792) arrived at a proposal, strongest first. +# `derive` is the odd one out: not "this is an instance of #N" but "this +# shape repeats with NO canon — derive one first" (note 2786's derive-first +# rule), so it carries a group key instead of a snippet. +PROPOSAL_BASES = ("symbol", "text", "reference", "signature", "semantic", "derive") + class CodeShape(Base, TimestampMixin): """One extracted code shape and its classification against canon (#2787). @@ -42,6 +49,20 @@ class CodeShape(Base, TimestampMixin): snippet_id is SET NULL on snippet deletion: the classification's target is gone but the judgment happened; the sync pass (step 2) re-files such rows as unclassified so they rejoin the todo instead of dangling. + + `signature` / `body_sha` (#2792) are the shape's content fingerprint — + its definition line and a whitespace/comment-insensitive hash of its + block — refreshed by every sync. They are what the mechanical proposer + matches on and what a later drift recheck compares against; the ledger + still never stores code bodies. + + The proposal columns hold the proposer's standing suggestion for an + UNCLASSIFIED row: `proposed_snippet_id` + `proposal_basis` + score for + "looks like an instance of #N", or `proposal_basis="derive"` + + `proposal_group` for "repeats with no canon". `proposed_sha` is the + body_sha the row was last examined at, so a refresh re-examines only + what changed. A judgment clears the proposal — the machine proposes, + judgment classifies. """ __tablename__ = "code_shapes" @@ -52,6 +73,7 @@ class CodeShape(Base, TimestampMixin): ), Index("ix_code_shapes_project_status", "project_id", "status"), Index("ix_code_shapes_snippet", "snippet_id"), + Index("ix_code_shapes_proposed", "project_id", "proposed_snippet_id"), ) id: Mapped[int] = mapped_column(primary_key=True) @@ -76,6 +98,31 @@ class CodeShape(Base, TimestampMixin): vanished_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True ) + signature: Mapped[str] = mapped_column(Text, default="") + body_sha: Mapped[str] = mapped_column(Text, default="") + proposed_snippet_id: Mapped[int | None] = mapped_column( + BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True + ) + proposal_basis: Mapped[str | None] = mapped_column(Text, nullable=True) + proposal_score: Mapped[float | None] = mapped_column(Float, nullable=True) + proposal_group: Mapped[str | None] = mapped_column(Text, nullable=True) + proposed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + proposed_sha: Mapped[str] = mapped_column(Text, default="") + + @property + def proposal(self) -> dict | None: + """The standing proposal as one object, or None when the proposer + has nothing to say about this row.""" + if self.proposed_snippet_id is None and not self.proposal_group: + return None + out: dict = {"basis": self.proposal_basis, "score": self.proposal_score} + if self.proposed_snippet_id is not None: + out["snippet_id"] = self.proposed_snippet_id + if self.proposal_group: + out["group"] = self.proposal_group + return out def to_dict(self) -> dict: return { @@ -93,6 +140,9 @@ class CodeShape(Base, TimestampMixin): "first_seen_commit": self.first_seen_commit, "last_seen_commit": self.last_seen_commit, "vanished_at": self.vanished_at.isoformat() if self.vanished_at else None, + "signature": self.signature, + "body_sha": self.body_sha, + "proposal": self.proposal, "created_at": self.created_at.isoformat(), "updated_at": self.updated_at.isoformat(), } diff --git a/src/scribe/services/backup.py b/src/scribe/services/backup.py index 97edde3..cd1c7ae 100644 --- a/src/scribe/services/backup.py +++ b/src/scribe/services/backup.py @@ -1163,6 +1163,11 @@ async def _restore_v2(data: dict) -> dict: first_seen_commit=cs_data.get("first_seen_commit", ""), last_seen_commit=cs_data.get("last_seen_commit", ""), vanished_at=_dt(cs_data["vanished_at"]) if cs_data.get("vanished_at") else None, + # Fingerprints restore; proposals (#2792) deliberately do not — + # they are mechanical, and the next refresh recomputes them + # against the restored snippet ids. + signature=cs_data.get("signature", ""), + body_sha=cs_data.get("body_sha", ""), created_at=_dt(cs_data.get("created_at")), updated_at=_dt(cs_data.get("updated_at")), )) diff --git a/src/scribe/services/coverage.py b/src/scribe/services/coverage.py index 04b13c0..8cbbb43 100644 --- a/src/scribe/services/coverage.py +++ b/src/scribe/services/coverage.py @@ -25,12 +25,14 @@ only ever reads the cache. """ from __future__ import annotations +import hashlib import io import json import logging import posixpath import re import tarfile +from typing import NamedTuple from datetime import datetime, timedelta, timezone from scribe.services.forge import ForgeSelector, get_forges @@ -91,6 +93,104 @@ _ARROW_RE = re.compile( ) +class Definition(NamedTuple): + """One extracted definition with its content fingerprint (#2792). + + `signature` is the definition line itself; `body_sha` hashes the block + whitespace- and comment-insensitively; `body` is the block's text, held + only for the duration of a refresh (the proposer matches on it) and + never stored. + """ + + kind: str + name: str + signature: str + body_sha: str + body: str + + +def _definition_on(raw: str) -> tuple[str, str] | None: + """The (kind, name) this one line defines, or None. First match wins — + the same order the hook's awk program tries.""" + m = _CSS_RE.match(raw) + if m: + return ("css", m.group(1)) + line = _MODIFIERS_RE.sub("", raw.lstrip()) + if m := _GO_METHOD_RE.match(line): + return ("sym", m.group(1)) + if m := _KEYWORD_RE.match(line): + name = m.group(1) + if name.startswith("__") and name.endswith("__"): + return None + return ("sym", name) + if m := _ARROW_RE.match(line): + return ("sym", m.group(1)) + return None + + +# A definition's block runs from its line until the next non-blank line at +# its own indentation or shallower that is not a closer — so a Python def ends +# at the next top-level statement, a braces block keeps its `}`, a CSS rule +# keeps its `}`. Capped so a generated monolith can't make one shape's +# fingerprint cover the file. +_BLOCK_CAP = 120 +_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);") +# Lines that don't change what a shape IS: comments and decorators. Dropped +# from the fingerprint so touching a comment above the next function doesn't +# read as this one's body changing. +_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "") +_SIGNATURE_CAP = 300 + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def _block_sha(lines: list[str]) -> str: + kept = [ + " ".join(ln.split()) + for ln in lines + if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES) + ] + return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16] + + +def extract_definitions(text: str) -> list[Definition]: + """Every definition this text makes, with signature + fingerprint. + + Duplicate (kind, name) within one text collapse to the first — the + ledger's identity is per file, so a second definition of the same name + (an overload, a re-declaration) is the same shape to it. + """ + lines = text.splitlines() + starts: list[tuple[int, str, str]] = [] + for i, raw in enumerate(lines): + hit = _definition_on(raw) + if hit: + starts.append((i, hit[0], hit[1])) + seen: set[tuple[str, str]] = set() + out: list[Definition] = [] + for i, kind, name in starts: + if (kind, name) in seen: + continue + seen.add((kind, name)) + base = _indent(lines[i]) + end = min(len(lines), i + _BLOCK_CAP) + for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)): + ln = lines[j] + if not ln.strip(): + continue + if _indent(ln) <= base and ln.strip() not in _CLOSERS: + end = j + break + block = lines[i:end] + out.append(Definition( + kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(block), + "\n".join(block), + )) + return out + + def extract_shapes(text: str) -> list[tuple[str, str]]: """Every (kind, name) this text DEFINES — kind is "css" or "sym". @@ -98,29 +198,7 @@ def extract_shapes(text: str) -> list[tuple[str, str]]: line, dunders are skipped (every class defines __init__ — guaranteed noise), duplicates within one text count once. """ - seen: set[tuple[str, str]] = set() - out: list[tuple[str, str]] = [] - for raw in text.splitlines(): - m = _CSS_RE.match(raw) - if m: - shape = ("css", m.group(1)) - else: - line = _MODIFIERS_RE.sub("", raw.lstrip()) - if m := _GO_METHOD_RE.match(line): - shape = ("sym", m.group(1)) - elif m := _KEYWORD_RE.match(line): - name = m.group(1) - if name.startswith("__") and name.endswith("__"): - continue - shape = ("sym", name) - elif m := _ARROW_RE.match(line): - shape = ("sym", m.group(1)) - else: - continue - if shape not in seen: - seen.add(shape) - out.append(shape) - return out + return [(d.kind, d.name) for d in extract_definitions(text)] def scannable(path: str) -> bool: @@ -131,14 +209,33 @@ def scannable(path: str) -> bool: return not path.lower().endswith(_SKIP_SUFFIXES) +class ArchiveShape(NamedTuple): + """A definition located in a repo archive — what the sync upserts and + the proposer matches. The leading (path, kind, name) triple is the + ledger identity; the rest is the fingerprint and the transient body.""" + + path: str + kind: str + name: str + signature: str + body_sha: str + body: str + + def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]: - """(path, kind, name) for every definition in a repo tarball. + """(path, kind, name) for every definition in a repo tarball — the + identity view of definitions_from_archive.""" + return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)] + + +def definitions_from_archive(blob: bytes) -> list[ArchiveShape]: + """Every definition in a repo tarball, with its fingerprint and body. Forge archives wrap content in a single top-level directory (repo-ref/); that component is stripped so paths match recorded snippet locations, which are repo-relative. Non-UTF-8 files are binaries and skipped. """ - shapes: list[tuple[str, str, str]] = [] + shapes: list[ArchiveShape] = [] with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar: for member in tar: if not member.isfile() or "/" not in member.name: @@ -153,7 +250,10 @@ def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]: text = handle.read().decode("utf-8") except UnicodeDecodeError: continue - shapes.extend((path, kind, name) for kind, name in extract_shapes(text)) + shapes.extend( + ArchiveShape(path, d.kind, d.name, d.signature, d.body_sha, d.body) + for d in extract_definitions(text) + ) return shapes @@ -253,13 +353,17 @@ async def compute_coverage( served: list[tuple[str, str]] = [] recorded = await _recorded_locations(user_id, project_id) + # The proposer's canon catalog, read once per refresh and shared across + # the project's repos (#2792). + canons = None + proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0} for key in await keys_for_project(user_id, project_id): hit = selector.resolve(key) if hit is None: continue # bound to a host no connection serves forge, api_repo = hit ref = await forge.default_branch(api_repo) - shapes = shapes_from_archive(await forge.archive(api_repo, ref)) + definitions = definitions_from_archive(await forge.archive(api_repo, ref)) # The head commit is provenance sugar on the ledger rows; failing to # learn it must not fail the sync — the ref names the point well # enough and the row timestamps carry the when. @@ -268,13 +372,31 @@ async def compute_coverage( except ForgeError: marker = ref await shape_ledger.sync_repo_shapes( - project_id, key, shapes, seen_marker=marker + project_id, key, definitions, seen_marker=marker ) served.append((key, ref)) + # Propose while the bodies are in hand — the one moment they exist. + # Canonical marking below only touches rows the proposer leaves + # alone (a canon's own location never gets a proposal), so the order + # is immaterial; the proposer must not be able to fail the refresh. + try: + if canons is None: + canons = await shape_ledger.canon_catalog(user_id) + stats = await shape_ledger.propose_for_repo( + user_id, project_id, key, definitions, canons=canons + ) + for k in proposer_stats: + proposer_stats[k] += stats.get(k, 0) + except Exception: + logger.warning("shape proposer failed for %s", key, exc_info=True) if not served: return None await shape_ledger.mark_canonicals(project_id, recorded) + try: + await shape_ledger.apply_derive_groups(project_id) + except Exception: + logger.warning("derive-first grouping failed", exc_info=True) # Project-wide readout, deliberately wider than this walk: a second bound # repo that was unreachable today still has live rows, and they count. @@ -290,11 +412,17 @@ async def compute_coverage( agg["accounted"] += row.status != "unclassified" unclassified = counts.pop("unclassified") + proposals = shape_ledger.proposal_summary(rows) return { "total": len(rows), "accounted": len(rows) - unclassified, "unclassified": unclassified, "counts": counts, + # The proposer's standing (#2792): canon proposals awaiting a + # confirm, the largest derive-first groups, and what this refresh did. + "proposed": proposals["proposed"], + "derive_groups": proposals["derive_groups"], + "proposer": proposer_stats, # Honesty flag, not decoration: every surface that shows the number # is expected to carry it through. "estimate": True, @@ -438,6 +566,14 @@ def coverage_line(coverage: dict) -> str: unclassified = coverage.get("unclassified", 0) if unclassified: line += f"; {unclassified} unclassified" + standing = [] + if coverage.get("proposed"): + standing.append(f"{coverage['proposed']} proposed") + n_groups = len(coverage.get("derive_groups") or []) + if n_groups: + standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}") + if standing: + line += f" ({', '.join(standing)})" gaps = [g["dir"] for g in coverage.get("largest_gaps") or []] if gaps: line += ", largest: " + ", ".join(gaps) diff --git a/src/scribe/services/shape_ledger.py b/src/scribe/services/shape_ledger.py index 2f7db0d..f214b1f 100644 --- a/src/scribe/services/shape_ledger.py +++ b/src/scribe/services/shape_ledger.py @@ -21,9 +21,11 @@ an extracted shape. """ from __future__ import annotations +import difflib import logging import re from datetime import datetime, timedelta, timezone +from typing import Iterable, NamedTuple from sqlalchemy import select @@ -65,15 +67,17 @@ def location_covers(loc_path: str, loc_symbol: str, path: str, name: str) -> boo async def sync_repo_shapes( project_id: int, repo_key: str, - shapes: list[tuple[str, str, str]], + shapes: list, *, seen_marker: str, ) -> None: - """Upsert one repo's extracted (path, kind, name) shapes into the ledger. + """Upsert one repo's extracted shapes into the ledger. - ``seen_marker`` is the commit the archive was read at when the forge can - say, else the ref name — provenance sugar; the row timestamps carry the - when. + ``shapes`` are (path, kind, name) triples, or the richer ArchiveShape + records (#2792) whose 4th/5th fields — signature, body_sha — refresh the + row's content fingerprint. ``seen_marker`` is the commit the archive was + read at when the forge can say, else the ref name — provenance sugar; + the row timestamps carry the when. """ now = datetime.now(timezone.utc) async with async_session() as session: @@ -87,7 +91,10 @@ async def sync_repo_shapes( ).scalars().all() by_key = {(r.path, r.symbol, r.kind): r for r in rows} seen: set[tuple[str, str, str]] = set() - for path, kind, name in shapes: + for shape in shapes: + path, kind, name = shape[0], shape[1], shape[2] + signature = shape[3] if len(shape) > 3 else "" + body_sha = shape[4] if len(shape) > 4 else "" key = (path, name, kind) if key in seen: continue @@ -98,9 +105,14 @@ async def sync_repo_shapes( project_id=project_id, repo_key=repo_key, path=path, symbol=name, kind=kind, first_seen_commit=seen_marker, last_seen_commit=seen_marker, + signature=signature, body_sha=body_sha, )) continue row.last_seen_commit = seen_marker + if signature: + row.signature = signature + if body_sha: + row.body_sha = body_sha # A shape that vanished and came back is live again — the vanish # stays visible in history via updated_at, not as a dead flag. row.vanished_at = None @@ -297,6 +309,10 @@ async def classify_shapes( status = item["status"] for row in matches: row.status = status + # The machine proposes, judgment classifies: any judgment + # retires the standing proposal; a withdrawal also forgets + # the examination so the next refresh proposes afresh. + _clear_proposal(row, reexamine=(status == "unclassified")) if status == "unclassified": row.snippet_id = None row.reason = None @@ -324,12 +340,16 @@ async def list_project_shapes( include_vanished: bool = False, limit: int = 100, offset: int = 0, + proposal: str = "", ) -> tuple[list[CodeShape], int]: """A filtered page of a project's ledger, with the unfiltered-match total. ([], 0) when the caller can't read the project — the same silence every other project list gives. ``path`` matches the exact file or anything - beneath it, mirroring recorded-location semantics. + beneath it, mirroring recorded-location semantics. ``proposal`` narrows + to rows the proposer has spoken about: "any", "canon" (an instance-of-#N + suggestion), "derive" (a repeats-with-no-canon group), or one basis + name (symbol/reference/text/signature/semantic). """ from sqlalchemy import func, or_ @@ -349,6 +369,17 @@ async def list_project_shapes( )) if snippet_id: conds.append(CodeShape.snippet_id == snippet_id) + if proposal == "any": + conds.append(or_( + CodeShape.proposed_snippet_id.isnot(None), + CodeShape.proposal_group.isnot(None), + )) + elif proposal == "canon": + conds.append(CodeShape.proposed_snippet_id.isnot(None)) + elif proposal == "derive": + conds.append(CodeShape.proposal_group.isnot(None)) + elif proposal: + conds.append(CodeShape.proposal_basis == proposal) async with async_session() as session: total = ( await session.execute( @@ -589,6 +620,7 @@ async def stamp_write_path_instances( row.reason = why row.classified_by = "hook" row.classified_at = now + _clear_proposal(row) stamped.append({ "path": path, "symbol": name, "kind": kind, "snippet_id": sid, "reason": why, @@ -596,3 +628,431 @@ async def stamp_write_path_instances( if stamped: await session.commit() return stamped + + +# --- the mechanical proposer (#2792): the machine proposes, judgment classifies +# +# Runs inside the coverage refresh — the one moment the shape BODIES exist +# (the archive is in memory; the ledger stores fingerprints, never code). +# Over every live unclassified row whose content changed since it was last +# examined, it tries, strongest first: +# +# symbol the shape bears a recorded canon's symbol at another location +# (a second definition of the canon's name — an instance or a +# duplicate to consolidate; either way, it answers to #N); +# text whitespace-insensitive containment either way between the +# body and the canon's recorded code (git-grep, in effect — +# a verbatim copy, or the canon's own call-site example); +# reference the body calls/uses a canon's symbol — the call-site shape +# the P7 backfill classified by hand (#2790); +# signature a sym whose definition line, name blanked, resembles the +# canon's (the move_*/resequence family shape); +# semantic the widest net: the body (concept-queried, as the write-path +# arm does) scores above the write-path threshold against a +# canon — capped per refresh, so the cost is bounded and a big +# ledger is worked through across refreshes. +# +# A hit is a PROPOSAL on the row (proposed_snippet_id/basis/score), never a +# classification: an agent confirms in batches (confirm_proposals) or judges +# otherwise (classify_shapes clears it). Rows with no canon hit are grouped +# by the derive-first rule (note 2786): the same body fingerprint in ≥2 +# places, or the same name defined in ≥3 files, is "a repeating shape with +# no canon — derive one first", carried as proposal_basis="derive" + a group +# key so sessions see the consolidation candidates as one thing. + +# Derive-first floors. Identical bodies twice is already a copy; a bare name +# needs more repetition before it reads as a family (setup/handler/register +# recur by convention, not by duplication). +_DERIVE_MIN_DUP = 2 +_DERIVE_MIN_NAME = 3 +# Semantic checks per repo per refresh — an embedding each (local fastembed), +# bounded so a 4,000-row ledger is worked through over refreshes, not in one. +_SEMANTIC_CAP = 150 +# Signature resemblance floor, name blanked (difflib ratio) — and a length +# floor, because `def NAME():` resembles `def NAME(x):` at 0.95 while saying +# nothing; a family shape has parameters to resemble. +_SIGNATURE_FLOOR = 0.8 +_SIGNATURE_MIN_LEN = 30 +# Textual containment needs enough substance to mean anything. +_TEXT_FLOOR = 40 +_BASIS_ORDER = ("symbol", "text", "reference", "signature") + + +class Canon(NamedTuple): + snippet_id: int + kind: str + symbol: str + locations: tuple[tuple[str, str], ...] + signature: str + code_norm: str + + +def _norm_text(text: str) -> str: + return " ".join((text or "").split()) + + +def signature_similarity(sig_a: str, name_a: str, sig_b: str, name_b: str) -> float: + """How alike two definition lines are once their own names are blanked — + `def move_event(project_id, event_id, after_id)` against + `def move_beat(project_id, beat_id, after_id)` reads high.""" + a = _norm_text(sig_a) + b = _norm_text(sig_b) + if name_a: + a = a.replace(name_a, "NAME") + if name_b: + b = b.replace(name_b, "NAME") + if len(a) < _SIGNATURE_MIN_LEN or len(b) < _SIGNATURE_MIN_LEN: + return 0.0 + return difflib.SequenceMatcher(None, a, b).ratio() + + +def text_contains(body: str, code: str) -> bool: + """Whitespace-insensitive containment, either way, above a substance + floor — the proposer's git-grep.""" + a = _norm_text(body) + b = _norm_text(code) + if len(a) < _TEXT_FLOOR or len(b) < _TEXT_FLOOR: + return False + return a in b or b in a + + +def match_canon( + kind: str, path: str, symbol: str, signature: str, body: str, + canons: Iterable[Canon], +) -> tuple[int, str, float] | None: + """The strongest (snippet_id, basis, score) a shape earns against the + canon catalog, by the non-semantic bases — or None. Strongest by + basis order, then by score within the basis.""" + best: dict[str, tuple[float, int]] = {} + + def offer(basis: str, score: float, sid: int) -> None: + cur = best.get(basis) + if cur is None or score > cur[0]: + best[basis] = (score, sid) + + norm_sym = _norm_symbol(symbol) + for c in canons: + if c.kind != kind: + continue + if c.symbol and _norm_symbol(c.symbol) == norm_sym: + if not any(location_covers(lp, ls, path, symbol) for lp, ls in c.locations): + offer("symbol", 1.0, c.snippet_id) + continue # its own location is canonical territory, not a proposal + if c.symbol and references_symbol(body, c.symbol, kind): + offer("reference", 0.9, c.snippet_id) + if c.code_norm and text_contains(body, c.code_norm): + offer("text", 0.95, c.snippet_id) + if kind == "sym" and c.signature: + ratio = signature_similarity(signature, symbol, c.signature, c.symbol) + if ratio >= _SIGNATURE_FLOOR: + offer("signature", round(ratio, 3), c.snippet_id) + for basis in _BASIS_ORDER: + if basis in best: + score, sid = best[basis] + return (sid, basis, score) + return None + + +async def canon_catalog(user_id: int) -> list[Canon]: + """Every snippet this user can browse, as matchable canon — own projects + and shared ones alike, because family canon counts (note 2786).""" + from scribe.models.note import Note + from scribe.services.access import browsable_notes_clause + from scribe.services.coverage import extract_definitions + from scribe.services.snippets import SNIPPET_NOTE_TYPE, snippet_fields + + async with async_session() as session: + notes = ( + await session.execute( + select(Note).where( + browsable_notes_clause(user_id), + Note.note_type == SNIPPET_NOTE_TYPE, + Note.deleted_at.is_(None), + ) + ) + ).scalars().all() + out: list[Canon] = [] + for note in notes: + fields = snippet_fields(note) + symbol = (fields.get("symbol") or "").strip() + kind = snippet_kind(symbol, fields.get("language") or "") + code = fields.get("code") or "" + defs = extract_definitions(code) + signature = "" + if defs: + own = next( + (d for d in defs if _norm_symbol(d.name) == _norm_symbol(symbol)), defs[0] + ) + signature = own.signature + out.append(Canon( + int(note.id), kind, symbol, + tuple( + ((loc.get("path") or ""), (loc.get("symbol") or "")) + for loc in fields.get("locations") or [] + ), + signature, _norm_text(code), + )) + return out + + +def _clear_proposal(row: CodeShape, *, reexamine: bool = False) -> None: + row.proposed_snippet_id = None + row.proposal_basis = None + row.proposal_score = None + row.proposal_group = None + if reexamine: + row.proposed_at = None + row.proposed_sha = "" + + +def _substance(text: str) -> int: + return len("".join((text or "").split())) + + +async def _semantic_canon( + user_id: int, body: str, allowed: set[int] +) -> tuple[int, float] | None: + from scribe.services.embeddings import semantic_search_notes + from scribe.services.plugin_context import ( + WRITEPATH_DEFAULT_THRESHOLD, WRITEPATH_MIN_CODE_CHARS, concept_query, + ) + + if _substance(body) < WRITEPATH_MIN_CODE_CHARS or not allowed: + return None + query = concept_query(body) or body + hits = await semantic_search_notes( + user_id, query, limit=3, threshold=WRITEPATH_DEFAULT_THRESHOLD, + note_type="snippet", scope="browse", + ) + for score, note in hits: + if int(note.id) in allowed: + return int(note.id), round(float(score), 3) + return None + + +async def propose_for_repo( + user_id: int, + project_id: int, + repo_key: str, + definitions: list, + *, + canons: list[Canon] | None = None, + semantic_cap: int = _SEMANTIC_CAP, +) -> dict: + """Examine one repo's live unclassified rows against canon and record + proposals. ``definitions`` are the ArchiveShape records the sync just + upserted (their bodies are the matching material). Rows whose fingerprint + is unchanged since their last examination are skipped; rows that only + the capped semantic pass could not reach stay unexamined, so the next + refresh reaches the next slice. Returns counts.""" + if canons is None: + canons = await canon_catalog(user_id) + by_key = {(d[0], d[1], d[2]): d for d in definitions} + sym_canon_ids = {c.snippet_id for c in canons if c.kind == "sym"} + now = datetime.now(timezone.utc) + examined = proposed = checked = 0 + async with async_session() as session: + rows = ( + await session.execute( + select(CodeShape).where( + CodeShape.project_id == project_id, + CodeShape.repo_key == repo_key, + CodeShape.status == "unclassified", + CodeShape.vanished_at.is_(None), + ) + ) + ).scalars().all() + semantic_todo: list[tuple[CodeShape, object]] = [] + for row in rows: + d = by_key.get((row.path, row.kind, row.symbol)) + if d is None: + continue + signature, body_sha, body = d[3], d[4], d[5] + if row.proposed_at is not None and row.proposed_sha == body_sha: + continue + examined += 1 + group = row.proposal_group # derive grouping is reassigned below + hit = match_canon(row.kind, row.path, row.symbol, signature, body, canons) + _clear_proposal(row) + row.proposal_group = group + row.proposed_at = now + row.proposed_sha = body_sha + if hit: + row.proposed_snippet_id, row.proposal_basis, row.proposal_score = hit + row.proposal_group = None + proposed += 1 + elif row.kind == "sym": + semantic_todo.append((row, d)) + for i, (row, d) in enumerate(semantic_todo): + if i >= semantic_cap: + # Not reached this refresh: leave it unexamined so the next + # refresh picks it up, rather than stamping a false "nothing". + row.proposed_at = None + row.proposed_sha = "" + continue + checked += 1 + try: + found = await _semantic_canon(user_id, d[5], sym_canon_ids) + except Exception: + logger.warning("semantic proposal failed", exc_info=True) + found = None + if found: + row.proposed_snippet_id, row.proposal_score = found + row.proposal_basis = "semantic" + row.proposal_group = None + proposed += 1 + await session.commit() + return {"examined": examined, "proposed": proposed, "semantic_checked": checked} + + +def derive_groups( + rows: Iterable[tuple[str, str, str, str]] +) -> dict[tuple[str, str, str], str]: + """The derive-first grouping over (path, kind, symbol, body_sha) rows + that matched no canon: {(path, kind, symbol): group_key}. Identical + bodies in ≥2 places group as `dup:`; the same name defined in ≥3 + files groups as `name::`; a row joins at most one group, + the copy before the name.""" + by_sha: dict[str, list[tuple[str, str, str]]] = {} + by_name: dict[tuple[str, str], list[tuple[str, str, str]]] = {} + for path, kind, symbol, sha in rows: + key = (path, kind, symbol) + if sha: + by_sha.setdefault(sha, []).append(key) + by_name.setdefault((kind, _norm_symbol(symbol)), []).append(key) + out: dict[tuple[str, str, str], str] = {} + for sha, keys in by_sha.items(): + if len(set(keys)) >= _DERIVE_MIN_DUP: + for key in keys: + out.setdefault(key, f"dup:{sha}") + for (kind, symbol), keys in by_name.items(): + if len({k[0] for k in keys}) >= _DERIVE_MIN_NAME: + for key in keys: + out.setdefault(key, f"name:{kind}:{symbol}") + return out + + +async def apply_derive_groups(project_id: int) -> int: + """Recompute derive-first groups over the project's live unclassified + rows that carry no canon proposal; returns how many rows are grouped.""" + now = datetime.now(timezone.utc) + grouped = 0 + async with async_session() as session: + rows = ( + await session.execute( + select(CodeShape).where( + CodeShape.project_id == project_id, + CodeShape.status == "unclassified", + CodeShape.vanished_at.is_(None), + CodeShape.proposed_snippet_id.is_(None), + ) + ) + ).scalars().all() + groups = derive_groups( + (r.path, r.kind, r.symbol, r.body_sha or "") for r in rows + ) + sizes: dict[str, int] = {} + for g in groups.values(): + sizes[g] = sizes.get(g, 0) + 1 + for row in rows: + key = groups.get((row.path, row.kind, row.symbol)) + if key: + row.proposal_basis = "derive" + row.proposal_group = key + row.proposal_score = float(sizes[key]) + if row.proposed_at is None: + row.proposed_at = now + grouped += 1 + elif row.proposal_group: + row.proposal_basis = None + row.proposal_group = None + row.proposal_score = None + await session.commit() + return grouped + + +def proposal_summary(rows: Iterable[CodeShape], *, top: int = 8) -> dict: + """The readout's view of the proposer's standing: how many canon + proposals await confirmation, and the largest derive-first groups.""" + proposed = 0 + groups: dict[str, dict] = {} + for row in rows: + if row.status != "unclassified": + continue + if row.proposed_snippet_id is not None: + proposed += 1 + elif row.proposal_group: + g = groups.setdefault(row.proposal_group, { + "group": row.proposal_group, "kind": row.kind, + "label": ( + ("." if row.kind == "css" else "") + row.symbol + if row.proposal_group.startswith("name:") + else f"{row.symbol} (identical body)" + ), + "size": 0, "paths": [], + }) + g["size"] += 1 + if len(g["paths"]) < 3: + g["paths"].append(row.path) + ranked = sorted(groups.values(), key=lambda g: (-g["size"], g["group"])) + return {"proposed": proposed, "derive_groups": ranked[:top]} + + +async def confirm_proposals( + user_id: int, + project_id: int, + *, + snippet_id: int = 0, + path: str = "", + basis: str = "", + min_score: float = 0.0, +) -> dict: + """Turn reviewed canon proposals into `instance` rows, in one batch. + + At least one of snippet_id / path / basis must narrow the batch — "confirm + everything proposed" without having looked is not a judgment. Each row + becomes instance-of-its-proposed-snippet, classified_by="agent", reason + naming the basis and score; the proposal is retired. Returns + {"confirmed": N}.""" + from scribe.services import access + + if not (snippet_id or path.strip() or basis.strip()): + raise ValueError( + "name what you reviewed: confirm by snippet_id, path, and/or basis" + ) + if not await access.can_write_project(user_id, project_id): + raise ValueError(f"project {project_id} not found or no write access") + from sqlalchemy import or_ + + conds = [ + CodeShape.project_id == project_id, + CodeShape.status == "unclassified", + CodeShape.vanished_at.is_(None), + CodeShape.proposed_snippet_id.isnot(None), + ] + if snippet_id: + conds.append(CodeShape.proposed_snippet_id == snippet_id) + if path.strip(): + clean = path.strip().strip("/") + conds.append(or_(CodeShape.path == clean, CodeShape.path.like(clean + "/%"))) + if basis.strip(): + conds.append(CodeShape.proposal_basis == basis.strip()) + now = datetime.now(timezone.utc) + confirmed = 0 + async with async_session() as session: + rows = (await session.execute(select(CodeShape).where(*conds))).scalars().all() + for row in rows: + if (row.proposal_score or 0.0) < min_score: + continue + row.status = "instance" + row.snippet_id = row.proposed_snippet_id + row.reason = ( + f"confirmed {row.proposal_basis} proposal" + f" ({(row.proposal_score or 0.0):.2f})" + ) + row.classified_by = "agent" + row.classified_at = now + _clear_proposal(row) + confirmed += 1 + await session.commit() + return {"confirmed": confirmed} diff --git a/tests/test_integration_shape_classify.py b/tests/test_integration_shape_classify.py index f96aefa..5a5aa42 100644 --- a/tests/test_integration_shape_classify.py +++ b/tests/test_integration_shape_classify.py @@ -351,3 +351,145 @@ async def test_recent_pulls_reads_the_usage_stream(seeded): assert sid in pulls assert sid + 1000 not in pulls assert sid + 2000 not in pulls + + +# --- #2792: the mechanical proposer against real rows ------------------------- + + +def _quiet_semantic(): + """The semantic basis needs the embedder; these tests prove the other + bases and the bookkeeping, so it answers "nothing" here.""" + from unittest.mock import AsyncMock, patch + + from scribe.services import shape_ledger + return patch.object(shape_ledger, "_semantic_canon", AsyncMock(return_value=None)) + + +def _defs(*items): + """ArchiveShape-like records: (path, kind, name, signature, body_sha, body).""" + import hashlib + out = [] + for path, kind, name, signature, body in items: + sha = hashlib.sha1(" ".join(body.split()).encode()).hexdigest()[:16] + out.append((path, kind, name, signature, sha, body)) + return out + + +@pytest.mark.integration +async def test_proposer_proposes_and_confirm_classifies(seeded): + """Bodies in hand, the proposer records proposals on unclassified rows — + symbol (a second `factory` elsewhere), reference (a call site), and + nothing for the unrelated — skips rows whose content it already judged, + and a scoped confirm turns proposals into agent instances while a + classify on another retires its proposal.""" + from scribe.services.shape_ledger import ( + confirm_proposals, propose_for_repo, + ) + + owner, other, pid, sid = ( + seeded["owner"], seeded["other"], seeded["pid"], seeded["snippet"] + ) + defs = _defs( + ("src/app.py", "sym", "make_app", "def make_app():", "def make_app():\n app = factory()\n return app"), + ("src/app.py", "sym", "Config", "class Config:", "class Config:\n debug = False"), + ("src/util.py", "sym", "helper", "def helper(x):", "def helper(x):\n return x"), + ("src/dup.py", "sym", "factory", "def factory():", "def factory():\n return 1"), + ("web/button.css", "css", "btn", ".btn {", ".btn {\n color: red;\n}"), + ) + await sync_repo_shapes(pid, REPO, defs, seen_marker="main") + + with _quiet_semantic(): + stats = await propose_for_repo(owner, pid, REPO, defs) + assert stats == {"examined": 5, "proposed": 2, "semantic_checked": 2} + rows, total = await list_project_shapes(owner, pid, proposal="canon") + by_symbol = {r.symbol: r for r in rows} + assert total == 2 + assert by_symbol["factory"].proposal == {"basis": "symbol", "score": 1.0, "snippet_id": sid} + assert by_symbol["make_app"].proposal == {"basis": "reference", "score": 0.9, "snippet_id": sid} + rows, _ = await list_project_shapes(owner, pid, proposal="reference") + assert [r.symbol for r in rows] == ["make_app"] + + # Same content again → nothing re-examined (the semantic cap would + # otherwise be spent on the same rows every refresh). A cap that leaves + # rows unreached leaves them UNexamined, so the next refresh gets them. + with _quiet_semantic(): + assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 0 + await classify_shapes(owner, pid, [ + {"path": "src/util.py", "symbol": "helper", "status": "unclassified"}, + ]) + assert (await propose_for_repo(owner, pid, REPO, defs, semantic_cap=0))["semantic_checked"] == 0 + assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 1 + + # Outsider can't confirm; the owner confirms by snippet, scoped. + with pytest.raises(ValueError): + await confirm_proposals(other, pid, snippet_id=sid) + assert await confirm_proposals(owner, pid, basis="symbol") == {"confirmed": 1} + rows, _ = await list_project_shapes(owner, pid, snippet_id=sid) + factory = next(r for r in rows if r.symbol == "factory") + assert factory.status == "instance" and factory.classified_by == "agent" + assert factory.reason == "confirmed symbol proposal (1.00)" + assert factory.proposal is None + + # A judgment on a proposed row retires the proposal; withdrawing a + # judgment forgets the examination so the next pass proposes afresh. + await classify_shapes(owner, pid, [ + {"path": "src/app.py", "symbol": "make_app", "status": "exempt", "reason": "bootstrap"}, + ]) + rows, _ = await list_project_shapes(owner, pid, path="src/app.py") + make_app = next(r for r in rows if r.symbol == "make_app") + assert make_app.status == "exempt" and make_app.proposal is None + await classify_shapes(owner, pid, [ + {"path": "src/app.py", "symbol": "make_app", "status": "unclassified"}, + ]) + with _quiet_semantic(): + assert (await propose_for_repo(owner, pid, REPO, defs))["proposed"] == 1 + rows, _ = await list_project_shapes(owner, pid, proposal="canon") + assert [r.symbol for r in rows] == ["make_app"] + + +@pytest.mark.integration +async def test_derive_groups_land_on_rows_and_in_the_summary(seeded): + """Shapes with no canon hit that repeat — identical bodies in two files, + the same name in three — carry a derive proposal, and the readout ranks + the families. A canon proposal keeps a row out of any derive group.""" + from scribe.services.shape_ledger import ( + apply_derive_groups, live_rows, propose_for_repo, proposal_summary, + ) + + owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"] + defs = _defs( + ("a/one.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"), + ("a/two.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"), + ("b/x.css", "css", "card", ".card {", ".card { padding: 1px }"), + ("b/y.css", "css", "card", ".card {", ".card { padding: 2px }"), + ("b/z.css", "css", "card", ".card {", ".card { padding: 3px }"), + ("c/only.py", "sym", "alone", "def alone():", "def alone():\n return 0"), + ("c/use.py", "sym", "boot", "def boot():", "def boot():\n return factory()"), + ) + await sync_repo_shapes(pid, REPO, defs, seen_marker="main") + with _quiet_semantic(): + await propose_for_repo(owner, pid, REPO, defs) + assert await apply_derive_groups(pid) == 5 + + rows, total = await list_project_shapes(owner, pid, proposal="derive") + assert total == 5 + groups = {(r.path, r.symbol): r.proposal for r in rows} + assert groups[("a/one.py", "slug")]["group"] == groups[("a/two.py", "slug")]["group"] + assert groups[("a/one.py", "slug")]["group"].startswith("dup:") + assert groups[("b/x.css", "card")] == {"basis": "derive", "score": 3.0, "group": "name:css:card"} + rows, _ = await list_project_shapes(owner, pid, proposal="any") + assert {r.symbol for r in rows} == {"slug", "card", "boot"} # boot: reference proposal + + summary = proposal_summary(await live_rows(pid)) + assert summary["proposed"] == 1 + assert [g["group"] for g in summary["derive_groups"]][0] == "name:css:card" + assert summary["derive_groups"][0]["label"] == ".card" + assert summary["derive_groups"][0]["size"] == 3 + + # One of the css copies gets judged → the group shrinks on the next pass. + await classify_shapes(owner, pid, [ + {"path": "b/z.css", "symbol": "card", "status": "exempt", "reason": "print sheet"}, + ]) + await apply_derive_groups(pid) + rows, _ = await list_project_shapes(owner, pid, proposal="derive") + assert {r.symbol for r in rows} == {"slug"} # 2 files < the name floor diff --git a/tests/test_pattern_coverage.py b/tests/test_pattern_coverage.py index f5822a1..8220527 100644 --- a/tests/test_pattern_coverage.py +++ b/tests/test_pattern_coverage.py @@ -456,3 +456,54 @@ async def test_unservable_binding_measures_nothing(seeded): await set_binding(uid, "https://github.com/somebody/else.git", other_pid) assert await compute_coverage(uid, other_pid, selector=_selector(_tarball(TREE))) is None + + +# --- #2792: fingerprints and the proposer's readout -------------------------- + + +def test_extract_definitions_fingerprints_each_block(): + """The block rule across the language families the extractor knows: a + Python def ends at the next top-level statement, a braces/CSS block keeps + its closer, and comments/decorators don't move the hash.""" + from scribe.services.coverage import extract_definitions + + text = ( + "import os\n\n" + "def a(x):\n # comment\n return x + 1\n\n\n" + "class B:\n def m(self):\n return 2\n\n" + ".btn {\n color: red;\n}\n" + "export const f = (x) => {\n return x;\n};\n" + ) + defs = {d.name: d for d in extract_definitions(text)} + assert set(defs) == {"a", "B", "m", "btn", "f"} + assert defs["a"].signature == "def a(x):" + assert defs["a"].body.startswith("def a(x):\n # comment\n return x + 1") + assert "class B" not in defs["a"].body + assert defs["B"].body.rstrip().endswith("return 2") + assert defs["btn"].body == ".btn {\n color: red;\n}" + assert defs["f"].body == "export const f = (x) => {\n return x;\n};" + assert all(len(d.body_sha) == 16 for d in defs.values()) + # Comment changes don't change what the shape IS; code changes do. + again = {d.name: d for d in extract_definitions(text.replace("# comment", "# other"))} + assert again["a"].body_sha == defs["a"].body_sha + changed = {d.name: d for d in extract_definitions(text.replace("x + 1", "x + 2"))} + assert changed["a"].body_sha != defs["a"].body_sha + # And the identity view is unchanged for the hook mirror. + from scribe.services.coverage import extract_shapes + assert extract_shapes(text) == [(d.kind, d.name) for d in extract_definitions(text)] + + +def test_coverage_line_names_the_proposers_standing(): + from scribe.services.coverage import coverage_line + + base = { + "total": 100, "accounted": 10, "unclassified": 90, + "counts": {"canonical": 10, "instance": 0, "variant": 0, "exempt": 0}, + "computed_at": "2026-08-21T00:00:00+00:00", + "largest_gaps": [{"dir": "src", "unclassified": 90, "total": 90}], + } + assert coverage_line(base).endswith("; 90 unclassified, largest: src") + line = coverage_line({**base, "proposed": 40, "derive_groups": [{"group": "a"}, {"group": "b"}]}) + assert "; 90 unclassified (40 proposed, 2 derive groups), largest: src" in line + line = coverage_line({**base, "proposed": 0, "derive_groups": [{"group": "a"}]}) + assert "(1 derive group)" in line diff --git a/tests/test_shape_ledger.py b/tests/test_shape_ledger.py index a176551..e98b45d 100644 --- a/tests/test_shape_ledger.py +++ b/tests/test_shape_ledger.py @@ -5,6 +5,8 @@ token-free serialisation. The sync pass (step 2) and the classification surface (step 3) grow their tests here; DB-backed behavior lands in the integration lane once there is behavior to exercise. """ +import pytest + from scribe.models import Base from scribe.models.code_shape import SHAPE_CLASSIFIERS, SHAPE_STATUSES, CodeShape @@ -135,3 +137,142 @@ def test_hook_is_a_server_internal_classifier(): assert "hook" in SHAPE_CLASSIFIERS assert "hook" not in _CALLER_VIAS + + +# --- step 6: the mechanical proposer (pure) --------------------------------- + + +def test_proposal_columns_and_vocabulary_are_pinned(): + """Fingerprints + the proposer's standing suggestion live on the row; the + basis vocabulary is fixed, with `derive` the odd one out (a group, not a + snippet).""" + from scribe.models.code_shape import PROPOSAL_BASES + + cols = CodeShape.__table__.c + for name in ("signature", "body_sha", "proposed_snippet_id", "proposal_basis", + "proposal_score", "proposal_group", "proposed_at", "proposed_sha"): + assert name in cols, name + fk = next(iter(cols.proposed_snippet_id.foreign_keys)) + assert fk.ondelete == "SET NULL" and fk.column.table.name == "notes" + assert "ix_code_shapes_proposed" in {ix.name for ix in CodeShape.__table__.indexes} + assert PROPOSAL_BASES == ("symbol", "text", "reference", "signature", "semantic", "derive") + + +def test_row_proposal_property_is_one_object_or_none(): + row = CodeShape(project_id=1, repo_key="r", path="a.py", symbol="f", kind="sym") + assert row.proposal is None + row.proposed_snippet_id, row.proposal_basis, row.proposal_score = 9, "symbol", 1.0 + assert row.proposal == {"basis": "symbol", "score": 1.0, "snippet_id": 9} + row.proposed_snippet_id = None + row.proposal_basis, row.proposal_group, row.proposal_score = "derive", "dup:abc", 3.0 + assert row.proposal == {"basis": "derive", "score": 3.0, "group": "dup:abc"} + + +def test_signature_similarity_blanks_the_names(): + from scribe.services.shape_ledger import signature_similarity as sim + + a = "def move_event(project_id: int, event_id: int, after_id: int | None):" + b = "def move_beat(project_id: int, beat_id: int, after_id: int | None):" + assert sim(a, "move_event", b, "move_beat") > 0.85 + assert sim(a, "move_event", "def export_pdf(manuscript, design, fonts):", "export_pdf") < 0.6 + assert sim("", "x", b, "move_beat") == 0.0 + # Trivial signatures resemble everything and mean nothing — floored out. + assert sim("def helper(x):", "helper", "def make_app():", "make_app") == 0.0 + + +def test_text_containment_is_whitespace_insensitive_with_a_floor(): + from scribe.services.shape_ledger import text_contains + + code = "const ok = await confirmed({ title: 'Delete?', confirmLabel: 'Delete' });\nif (!ok) return;" + body = "async function onDelete() {\n const ok = await confirmed({\n title: 'Delete?',\n confirmLabel: 'Delete'\n });\n if (!ok) return;\n}" + assert text_contains(body, code) + assert text_contains(code, body) + assert not text_contains("x = 1", "x = 1") # below the substance floor + + +def _canon(sid, kind="sym", symbol="", locations=(), signature="", code=""): + from scribe.services.shape_ledger import Canon, _norm_text + return Canon(sid, kind, symbol, tuple(locations), signature, _norm_text(code)) + + +def test_match_canon_orders_bases_strongest_first_and_respects_kind(): + from scribe.services.shape_ledger import match_canon + + confirmed = _canon( + 7, "sym", "confirmed", [("frontend/src/composables/useConfirm.ts", "confirmed")], + "export async function confirmed(opts: ConfirmOptions): Promise {", + "export async function confirmed(opts: ConfirmOptions): Promise { /* singleton */ }", + ) + mover = _canon( + 8, "sym", "move_beat", [("src/forge/plot.py", "move_beat")], + "def move_beat(project_id: int, beat_id: int, after_id: int | None) -> None:", + ) + btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")], + ".btn-primary {", ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }") + canons = [confirmed, mover, btn] + + # symbol: a second `confirmed` defined elsewhere answers to #7 — but the + # canon's own location never does (that row is canonical, not a proposal). + assert match_canon("sym", "src/other.ts", "confirmed", "function confirmed() {", "", canons) == (7, "symbol", 1.0) + assert match_canon("sym", "frontend/src/composables/useConfirm.ts", "confirmed", + "export async function confirmed(", "", canons) is None + # reference: a call site of the canon. + body = "async function onTrash() {\n const ok = await confirmed({ title: 'x' });\n if (!ok) return;\n}" + assert match_canon("sym", "c.vue", "onTrash", "async function onTrash() {", body, canons) == (7, "reference", 0.9) + # signature: the family shape, names blanked. + hit = match_canon("sym", "src/forge/timeline.py", "move_event", + "def move_event(project_id: int, event_id: int, after_id: int | None) -> None:", + " pass", canons) + assert hit and hit[0] == 8 and hit[1] == "signature" and hit[2] >= 0.8 + # text: the canon's code contains the shape's body (a css copy), kind-matched — + # the same text as a `sym` shape matches no css canon. + css_body = ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }" + assert match_canon("css", "web/other.css", "btn-big", ".btn-big {", css_body, canons) == (9, "text", 0.95) + assert match_canon("sym", "web/other.css", "btn-big", ".btn-big {", css_body, canons) is None + # nothing in play + assert match_canon("sym", "x.py", "unrelated", "def unrelated(a, b, c, d, e):", "return 1", canons) is None + + +def test_match_canon_symbol_beats_everything_including_css_copies(): + """The previous test's css `btn-primary`-elsewhere case, stated plainly: + a second definition of the canon's own name is the symbol basis.""" + from scribe.services.shape_ledger import match_canon + btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")], ".btn-primary {", ".btn-primary { color: red; padding: 4px 8px; border-radius: 4px; }") + assert match_canon("css", "web/other.css", "btn-primary", ".btn-primary {", ".btn-primary { color: blue }", [btn]) == (9, "symbol", 1.0) + + +def test_derive_groups_copy_before_name_with_floors(): + from scribe.services.shape_ledger import derive_groups + + rows = [ + ("a.py", "sym", "helper", "sha1"), ("b.py", "sym", "helper", "sha1"), # identical copies + ("c.py", "sym", "helper", "sha9"), # same name, 3rd file + ("d.css", "css", "btn", "s1"), ("e.css", "css", "btn", "s2"), ("f.css", "css", "btn", "s3"), + ("g.py", "sym", "main", "s4"), ("h.py", "sym", "main", "s5"), # only 2 files → no name group + ("i.py", "sym", "one", "s6"), + ] + g = derive_groups(rows) + assert g[("a.py", "sym", "helper")] == "dup:sha1" == g[("b.py", "sym", "helper")] + assert g[("c.py", "sym", "helper")] == "name:sym:helper" + assert g[("d.css", "css", "btn")] == "name:css:btn" + assert ("g.py", "sym", "main") not in g + assert ("i.py", "sym", "one") not in g + + +def test_confirm_requires_a_named_scope(): + import asyncio + + from scribe.services.shape_ledger import confirm_proposals + + with pytest.raises(ValueError) as err: + asyncio.run(confirm_proposals(1, 2)) + assert "name what you reviewed" in str(err.value) + + +def test_proposer_tools_are_mounted(): + from scribe.mcp.server import build_mcp_server + + mcp = build_mcp_server() + assert mcp._tool_manager.get_tool("confirm_shape_proposals") is not None + tool = mcp._tool_manager.get_tool("list_shapes") + assert "proposal" in tool.parameters.get("properties", {})