feat(ledger): mechanical proposer — every refresh proposes instances against canon and groups derive-first candidates; agents confirm in batches (#2792, milestone 294 step 6)
CI & Build / Python lint (push) Successful in 5s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Failing after 34s
CI & Build / TypeScript typecheck (push) Successful in 35s
CI & Build / Python tests (push) Successful in 1m17s
CI & Build / Build & push image (push) Successful in 48s

Shapes now carry a content fingerprint (signature + whitespace/comment-
insensitive body_sha; migration 0080) and the proposer runs inside the
coverage refresh, the one moment bodies exist: symbol elsewhere → textual
containment → body references the canon → signature resemblance → semantic
(capped per refresh, unreached rows stay unexamined for the next). A hit is
a proposal on the row (proposed_snippet_id/basis/score), never a
classification; rows with no canon hit group by the derive-first rule
(identical body in ≥2 places, same name in ≥3 files) as proposal_basis=
derive + a group key. list_shapes(proposal=any|canon|derive|<basis>) is the
queue; confirm_shape_proposals(project_id, snippet_id|path|basis) confirms
in batches as agent instances; any classify_shapes/hook stamp retires the
proposal. Readout carries proposed + derive_groups (line, payload, card).
Plugin 0.1.35 (skill: the machine proposes, judgment classifies).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-20 21:30:35 -04:00
co-authored by Claude Fable 5
parent a9e1cddba7
commit ba0030e51d
12 changed files with 1190 additions and 43 deletions
@@ -0,0 +1,54 @@
"""Shape fingerprints + the mechanical proposer's columns (#2792, milestone 294)
Revision ID: 0080
Revises: 0079
Create Date: 2026-08-21
Two additions to the ledger. `signature` / `body_sha` fingerprint each shape
(definition line + a whitespace/comment-insensitive hash of its block) so the
proposer can match on content and a later drift recheck can notice change,
without the ledger ever storing code. The proposal columns carry the
proposer's standing suggestion for an unclassified row — instance-of-#N with
a basis and score, or a derive-first group key — and `proposed_sha`
remembers the content it was judged at so a refresh re-examines only what
changed. Mechanical and recomputable: a restore that lacks them loses
nothing the next refresh does not rebuild.
"""
import sqlalchemy as sa
from alembic import op
revision = "0080"
down_revision = "0079"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("code_shapes", sa.Column("signature", sa.Text(), nullable=False, server_default=""))
op.add_column("code_shapes", sa.Column("body_sha", sa.Text(), nullable=False, server_default=""))
op.add_column(
"code_shapes",
sa.Column(
"proposed_snippet_id",
sa.BigInteger(),
sa.ForeignKey("notes.id", ondelete="SET NULL"),
nullable=True,
),
)
op.add_column("code_shapes", sa.Column("proposal_basis", sa.Text(), nullable=True))
op.add_column("code_shapes", sa.Column("proposal_score", sa.Float(), nullable=True))
op.add_column("code_shapes", sa.Column("proposal_group", sa.Text(), nullable=True))
op.add_column("code_shapes", sa.Column("proposed_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("code_shapes", sa.Column("proposed_sha", sa.Text(), nullable=False, server_default=""))
op.create_index(
"ix_code_shapes_proposed", "code_shapes", ["project_id", "proposed_snippet_id"]
)
def downgrade() -> None:
op.drop_index("ix_code_shapes_proposed", table_name="code_shapes")
for col in (
"proposed_sha", "proposed_at", "proposal_group", "proposal_score",
"proposal_basis", "proposed_snippet_id", "body_sha", "signature",
):
op.drop_column("code_shapes", col)
+29
View File
@@ -427,6 +427,13 @@ interface CoverageGap {
unclassified: number;
total: number;
}
interface DeriveGroup {
group: string;
kind: string;
label: string;
size: number;
paths: string[];
}
interface Coverage {
total: number;
accounted: number;
@@ -436,6 +443,10 @@ interface Coverage {
computed_at: string;
repos: { repo: string; ref: string; total: number; accounted: number }[];
largest_gaps: CoverageGap[];
// The mechanical proposer's standing (#2792): canon proposals awaiting an
// agent's confirm, and the biggest repeats-with-no-canon families.
proposed?: number;
derive_groups?: DeriveGroup[];
}
const coverage = ref<Coverage | null>(null);
@@ -746,6 +757,24 @@ async function confirmDelete() {
{{ gap.dir }} <span class="coverage-gap-count">{{ gap.unclassified }}</span>
</span>
</div>
<div
v-if="coverage.proposed || coverage.derive_groups?.length"
class="coverage-gaps"
title="The proposer matched these against canon; an agent confirms them in batches (confirm_shape_proposals)."
>
<span class="coverage-gaps-label">Proposed:</span>
<span v-if="coverage.proposed" class="coverage-gap-chip">
{{ coverage.proposed }} awaiting confirm
</span>
<span
v-for="g in coverage.derive_groups || []"
:key="g.group"
class="coverage-gap-chip"
:title="'Repeats with no canon — derive one first. ' + g.paths.join(', ')"
>
{{ g.label }} <span class="coverage-gap-count">×{{ g.size }}</span>
</span>
</div>
</template>
<p v-else class="coverage-empty">
Not measured yet Refresh reads the bound repo's definitions into
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "scribe",
"description": "Scribe system-of-record for Claude Code: MCP tools over your notes/tasks/projects/rules, a session-start push channel that surfaces your always-on rules + active-project context, process-skills (writing-plans, systematic-debugging, verification, brainstorming, reusing-code), and your saved Scribe Processes auto-surfaced as skills (/scribe:sync). Replaces superpowers + file-memory with one app-backed plugin.",
"version": "0.1.34",
"version": "0.1.35",
"author": { "name": "Bryan Van Deusen" },
"mcpServers": {
"scribe": {
+21
View File
@@ -51,6 +51,27 @@ need a hand judgment:
judgment: it never overrides a classification you made, and a
`classify_shapes` call overrides it.
## The machine proposes, judgment classifies
Every coverage refresh runs the **mechanical proposer** over the unclassified
rows: same symbol as a canon elsewhere → textual containment → body
references a canon → signature resemblance → semantic (capped per refresh). A hit
is a *proposal* on the row, never a classification. Work the queue in bulk:
1. `list_shapes(project_id, proposal="canon", snippet_id=N)` or
`path="dir"` — read the page; `proposal` carries snippet_id, basis, score.
2. `confirm_shape_proposals(project_id, snippet_id=N)` (or `path=`,
`basis=`) for the ones that hold — hundreds at a time; `symbol` and
`reference` proposals are near-certain, `semantic` deserves a look.
3. `classify_shapes` the rest — variant, exempt, or instance of a different
snippet. Any judgment retires the proposal.
`list_shapes(project_id, proposal="derive")` lists the **derive-first
candidates** — the same body in ≥2 places or the same name defined in ≥3
files, with no canon at all (`proposal.group` names the family; the coverage
payload's `derive_groups` ranks the biggest). That is the consolidation
queue, not a classification queue: see below.
## The derive-first rule
N same-shaped occurrences matching **no** recorded canon is never N loose
+65 -7
View File
@@ -62,6 +62,7 @@ async def list_shapes(
include_vanished: bool = False,
limit: int = 100,
offset: int = 0,
proposal: str = "",
) -> dict:
"""Read a project's shape ledger — `status="unclassified"` IS the todo.
@@ -75,6 +76,11 @@ async def list_shapes(
snippet_id: rows classified against this snippet — a consumer map.
include_vanished: include shapes no longer in the tree (history).
limit/offset: page through big ledgers (limit caps at 500).
proposal: the proposer's queue (#2792) — "any", "canon" (rows the
machine thinks are an instance of a snippet: `proposal` carries
snippet_id, basis, score), "derive" (rows that repeat with NO
canon: `proposal.group` names the family), or one basis
(symbol/text/reference/signature/semantic).
Returns {"shapes": [...], "total": N} — total counts every match, not
just this page. Each row's `classified_by` says who judged: agent /
@@ -84,20 +90,61 @@ async def list_shapes(
was stamped an instance with the evidence in `reason`. A hook row is
overridable by any classify_shapes call; it never overrides yours.
Classify what you can judge with classify_shapes; a repeating shape
with NO recorded canon is a derive-one-first moment (consolidate onto a
reference, create_snippet it, then classify the rest against it), never
N loose classifications.
THE FAST PATH through a big todo is the proposer's queue: every coverage
refresh matches unclassified shapes against canon (strongest basis
first: same symbol elsewhere → textual containment → body references
the canon → signature resemblance → semantic) and attaches a
`proposal` to each row it can speak for. Review `proposal="canon"` by
snippet or directory, then confirm_shape_proposals the ones that hold —
hundreds at a time — and classify_shapes the rest (variant/exempt, or
instance of a different snippet). `proposal="derive"` lists the
derive-first candidates: a repeating shape with NO recorded canon is
never N loose classifications — consolidate onto a reference,
create_snippet it, then classify the group against it.
"""
uid = current_user_id()
rows, total = await shape_ledger_svc.list_project_shapes(
uid, project_id,
status=status, path=path, snippet_id=snippet_id,
include_vanished=include_vanished, limit=limit, offset=offset,
proposal=proposal,
)
return {"shapes": [r.to_dict() for r in rows], "total": total}
async def confirm_shape_proposals(
project_id: int,
snippet_id: int = 0,
path: str = "",
basis: str = "",
min_score: float = 0.0,
) -> dict:
"""Confirm the proposer's canon proposals you have reviewed, in batch.
The machine proposes, judgment classifies (#2792): each matching row —
live, unclassified, carrying a `proposal` with a snippet_id — becomes
`instance` of that snippet, classified_by="agent", reason naming the
basis and score. Narrow to what you actually looked at: at least one of
snippet_id (confirm one canon's whole queue after reading its
`list_shapes(proposal="canon", ...)` page), path (a directory you
audited), or basis (e.g. "symbol" and "reference" are near-certain;
"semantic" deserves a look first) is required — a bare confirm-all is
not a judgment. min_score trims a basis's tail.
Proposals you do NOT confirm are judged with classify_shapes (variant,
exempt, or instance of a different snippet) — any judgment retires the
proposal. Requires write access. Returns {"confirmed": N}.
"""
uid = current_user_id()
try:
return await shape_ledger_svc.confirm_proposals(
uid, project_id, snippet_id=snippet_id, path=path, basis=basis,
min_score=min_score,
)
except ValueError as exc:
return {"error": str(exc)}
async def refresh_pattern_coverage(project_id: int) -> dict:
"""Seed or refresh the project's shape ledger NOW, and return the readout.
@@ -114,9 +161,17 @@ async def refresh_pattern_coverage(project_id: int) -> dict:
owner adds one (Settings → Integrations → Git Forges); no served repo →
bind_repo on a host a connection serves.
The refresh is also when the mechanical proposer runs (#2792): with the
repo bodies in hand it matches every changed unclassified shape against
canon and records proposals (see list_shapes proposal=), then regroups
the derive-first candidates. Semantic matching is capped per refresh, so
a large ledger's queue grows across refreshes rather than in one.
Returns the accounting payload — total, accounted, counts by status,
unclassified, repos, largest_gaps — plus `pattern_coverage`, the same
one-line summary enter_project carries.
unclassified, repos, largest_gaps, `proposed` (canon proposals awaiting
confirmation), `derive_groups` (the biggest repeats-with-no-canon
families), `proposer` (what this refresh examined) — plus
`pattern_coverage`, the same one-line summary enter_project carries.
"""
uid = current_user_id()
coverage = await coverage_svc.refresh_for_caller(uid, project_id)
@@ -127,5 +182,8 @@ async def refresh_pattern_coverage(project_id: int) -> dict:
def register(mcp) -> None:
for fn in (classify_shapes, list_shapes, refresh_pattern_coverage):
for fn in (
classify_shapes, list_shapes, refresh_pattern_coverage,
confirm_shape_proposals,
):
mcp.tool(name=fn.__name__)(fn)
+50
View File
@@ -3,6 +3,7 @@ from datetime import datetime
from sqlalchemy import (
BigInteger,
DateTime,
Float,
ForeignKey,
Index,
Integer,
@@ -19,6 +20,12 @@ from scribe.models.base import TimestampMixin
SHAPE_STATUSES = ("canonical", "instance", "variant", "exempt", "unclassified")
SHAPE_CLASSIFIERS = ("agent", "audit", "hook", "mechanical", "import")
# How the mechanical proposer (#2792) arrived at a proposal, strongest first.
# `derive` is the odd one out: not "this is an instance of #N" but "this
# shape repeats with NO canon — derive one first" (note 2786's derive-first
# rule), so it carries a group key instead of a snippet.
PROPOSAL_BASES = ("symbol", "text", "reference", "signature", "semantic", "derive")
class CodeShape(Base, TimestampMixin):
"""One extracted code shape and its classification against canon (#2787).
@@ -42,6 +49,20 @@ class CodeShape(Base, TimestampMixin):
snippet_id is SET NULL on snippet deletion: the classification's target
is gone but the judgment happened; the sync pass (step 2) re-files such
rows as unclassified so they rejoin the todo instead of dangling.
`signature` / `body_sha` (#2792) are the shape's content fingerprint —
its definition line and a whitespace/comment-insensitive hash of its
block — refreshed by every sync. They are what the mechanical proposer
matches on and what a later drift recheck compares against; the ledger
still never stores code bodies.
The proposal columns hold the proposer's standing suggestion for an
UNCLASSIFIED row: `proposed_snippet_id` + `proposal_basis` + score for
"looks like an instance of #N", or `proposal_basis="derive"` +
`proposal_group` for "repeats with no canon". `proposed_sha` is the
body_sha the row was last examined at, so a refresh re-examines only
what changed. A judgment clears the proposal — the machine proposes,
judgment classifies.
"""
__tablename__ = "code_shapes"
@@ -52,6 +73,7 @@ class CodeShape(Base, TimestampMixin):
),
Index("ix_code_shapes_project_status", "project_id", "status"),
Index("ix_code_shapes_snippet", "snippet_id"),
Index("ix_code_shapes_proposed", "project_id", "proposed_snippet_id"),
)
id: Mapped[int] = mapped_column(primary_key=True)
@@ -76,6 +98,31 @@ class CodeShape(Base, TimestampMixin):
vanished_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
signature: Mapped[str] = mapped_column(Text, default="")
body_sha: Mapped[str] = mapped_column(Text, default="")
proposed_snippet_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
proposal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
proposal_score: Mapped[float | None] = mapped_column(Float, nullable=True)
proposal_group: Mapped[str | None] = mapped_column(Text, nullable=True)
proposed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
proposed_sha: Mapped[str] = mapped_column(Text, default="")
@property
def proposal(self) -> dict | None:
"""The standing proposal as one object, or None when the proposer
has nothing to say about this row."""
if self.proposed_snippet_id is None and not self.proposal_group:
return None
out: dict = {"basis": self.proposal_basis, "score": self.proposal_score}
if self.proposed_snippet_id is not None:
out["snippet_id"] = self.proposed_snippet_id
if self.proposal_group:
out["group"] = self.proposal_group
return out
def to_dict(self) -> dict:
return {
@@ -93,6 +140,9 @@ class CodeShape(Base, TimestampMixin):
"first_seen_commit": self.first_seen_commit,
"last_seen_commit": self.last_seen_commit,
"vanished_at": self.vanished_at.isoformat() if self.vanished_at else None,
"signature": self.signature,
"body_sha": self.body_sha,
"proposal": self.proposal,
"created_at": self.created_at.isoformat(),
"updated_at": self.updated_at.isoformat(),
}
+5
View File
@@ -1163,6 +1163,11 @@ async def _restore_v2(data: dict) -> dict:
first_seen_commit=cs_data.get("first_seen_commit", ""),
last_seen_commit=cs_data.get("last_seen_commit", ""),
vanished_at=_dt(cs_data["vanished_at"]) if cs_data.get("vanished_at") else None,
# Fingerprints restore; proposals (#2792) deliberately do not —
# they are mechanical, and the next refresh recomputes them
# against the restored snippet ids.
signature=cs_data.get("signature", ""),
body_sha=cs_data.get("body_sha", ""),
created_at=_dt(cs_data.get("created_at")),
updated_at=_dt(cs_data.get("updated_at")),
))
+164 -28
View File
@@ -25,12 +25,14 @@ only ever reads the cache.
"""
from __future__ import annotations
import hashlib
import io
import json
import logging
import posixpath
import re
import tarfile
from typing import NamedTuple
from datetime import datetime, timedelta, timezone
from scribe.services.forge import ForgeSelector, get_forges
@@ -91,6 +93,104 @@ _ARROW_RE = re.compile(
)
class Definition(NamedTuple):
"""One extracted definition with its content fingerprint (#2792).
`signature` is the definition line itself; `body_sha` hashes the block
whitespace- and comment-insensitively; `body` is the block's text, held
only for the duration of a refresh (the proposer matches on it) and
never stored.
"""
kind: str
name: str
signature: str
body_sha: str
body: str
def _definition_on(raw: str) -> tuple[str, str] | None:
"""The (kind, name) this one line defines, or None. First match wins —
the same order the hook's awk program tries."""
m = _CSS_RE.match(raw)
if m:
return ("css", m.group(1))
line = _MODIFIERS_RE.sub("", raw.lstrip())
if m := _GO_METHOD_RE.match(line):
return ("sym", m.group(1))
if m := _KEYWORD_RE.match(line):
name = m.group(1)
if name.startswith("__") and name.endswith("__"):
return None
return ("sym", name)
if m := _ARROW_RE.match(line):
return ("sym", m.group(1))
return None
# A definition's block runs from its line until the next non-blank line at
# its own indentation or shallower that is not a closer — so a Python def ends
# at the next top-level statement, a braces block keeps its `}`, a CSS rule
# keeps its `}`. Capped so a generated monolith can't make one shape's
# fingerprint cover the file.
_BLOCK_CAP = 120
_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);")
# Lines that don't change what a shape IS: comments and decorators. Dropped
# from the fingerprint so touching a comment above the next function doesn't
# read as this one's body changing.
_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "<!--", "-->")
_SIGNATURE_CAP = 300
def _indent(line: str) -> int:
return len(line) - len(line.lstrip())
def _block_sha(lines: list[str]) -> str:
kept = [
" ".join(ln.split())
for ln in lines
if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES)
]
return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16]
def extract_definitions(text: str) -> list[Definition]:
"""Every definition this text makes, with signature + fingerprint.
Duplicate (kind, name) within one text collapse to the first — the
ledger's identity is per file, so a second definition of the same name
(an overload, a re-declaration) is the same shape to it.
"""
lines = text.splitlines()
starts: list[tuple[int, str, str]] = []
for i, raw in enumerate(lines):
hit = _definition_on(raw)
if hit:
starts.append((i, hit[0], hit[1]))
seen: set[tuple[str, str]] = set()
out: list[Definition] = []
for i, kind, name in starts:
if (kind, name) in seen:
continue
seen.add((kind, name))
base = _indent(lines[i])
end = min(len(lines), i + _BLOCK_CAP)
for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)):
ln = lines[j]
if not ln.strip():
continue
if _indent(ln) <= base and ln.strip() not in _CLOSERS:
end = j
break
block = lines[i:end]
out.append(Definition(
kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(block),
"\n".join(block),
))
return out
def extract_shapes(text: str) -> list[tuple[str, str]]:
"""Every (kind, name) this text DEFINES — kind is "css" or "sym".
@@ -98,29 +198,7 @@ def extract_shapes(text: str) -> list[tuple[str, str]]:
line, dunders are skipped (every class defines __init__ — guaranteed
noise), duplicates within one text count once.
"""
seen: set[tuple[str, str]] = set()
out: list[tuple[str, str]] = []
for raw in text.splitlines():
m = _CSS_RE.match(raw)
if m:
shape = ("css", m.group(1))
else:
line = _MODIFIERS_RE.sub("", raw.lstrip())
if m := _GO_METHOD_RE.match(line):
shape = ("sym", m.group(1))
elif m := _KEYWORD_RE.match(line):
name = m.group(1)
if name.startswith("__") and name.endswith("__"):
continue
shape = ("sym", name)
elif m := _ARROW_RE.match(line):
shape = ("sym", m.group(1))
else:
continue
if shape not in seen:
seen.add(shape)
out.append(shape)
return out
return [(d.kind, d.name) for d in extract_definitions(text)]
def scannable(path: str) -> bool:
@@ -131,14 +209,33 @@ def scannable(path: str) -> bool:
return not path.lower().endswith(_SKIP_SUFFIXES)
class ArchiveShape(NamedTuple):
"""A definition located in a repo archive — what the sync upserts and
the proposer matches. The leading (path, kind, name) triple is the
ledger identity; the rest is the fingerprint and the transient body."""
path: str
kind: str
name: str
signature: str
body_sha: str
body: str
def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
"""(path, kind, name) for every definition in a repo tarball.
"""(path, kind, name) for every definition in a repo tarball — the
identity view of definitions_from_archive."""
return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)]
def definitions_from_archive(blob: bytes) -> list[ArchiveShape]:
"""Every definition in a repo tarball, with its fingerprint and body.
Forge archives wrap content in a single top-level directory (repo-ref/);
that component is stripped so paths match recorded snippet locations,
which are repo-relative. Non-UTF-8 files are binaries and skipped.
"""
shapes: list[tuple[str, str, str]] = []
shapes: list[ArchiveShape] = []
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
for member in tar:
if not member.isfile() or "/" not in member.name:
@@ -153,7 +250,10 @@ def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
text = handle.read().decode("utf-8")
except UnicodeDecodeError:
continue
shapes.extend((path, kind, name) for kind, name in extract_shapes(text))
shapes.extend(
ArchiveShape(path, d.kind, d.name, d.signature, d.body_sha, d.body)
for d in extract_definitions(text)
)
return shapes
@@ -253,13 +353,17 @@ async def compute_coverage(
served: list[tuple[str, str]] = []
recorded = await _recorded_locations(user_id, project_id)
# The proposer's canon catalog, read once per refresh and shared across
# the project's repos (#2792).
canons = None
proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0}
for key in await keys_for_project(user_id, project_id):
hit = selector.resolve(key)
if hit is None:
continue # bound to a host no connection serves
forge, api_repo = hit
ref = await forge.default_branch(api_repo)
shapes = shapes_from_archive(await forge.archive(api_repo, ref))
definitions = definitions_from_archive(await forge.archive(api_repo, ref))
# The head commit is provenance sugar on the ledger rows; failing to
# learn it must not fail the sync — the ref names the point well
# enough and the row timestamps carry the when.
@@ -268,13 +372,31 @@ async def compute_coverage(
except ForgeError:
marker = ref
await shape_ledger.sync_repo_shapes(
project_id, key, shapes, seen_marker=marker
project_id, key, definitions, seen_marker=marker
)
served.append((key, ref))
# Propose while the bodies are in hand — the one moment they exist.
# Canonical marking below only touches rows the proposer leaves
# alone (a canon's own location never gets a proposal), so the order
# is immaterial; the proposer must not be able to fail the refresh.
try:
if canons is None:
canons = await shape_ledger.canon_catalog(user_id)
stats = await shape_ledger.propose_for_repo(
user_id, project_id, key, definitions, canons=canons
)
for k in proposer_stats:
proposer_stats[k] += stats.get(k, 0)
except Exception:
logger.warning("shape proposer failed for %s", key, exc_info=True)
if not served:
return None
await shape_ledger.mark_canonicals(project_id, recorded)
try:
await shape_ledger.apply_derive_groups(project_id)
except Exception:
logger.warning("derive-first grouping failed", exc_info=True)
# Project-wide readout, deliberately wider than this walk: a second bound
# repo that was unreachable today still has live rows, and they count.
@@ -290,11 +412,17 @@ async def compute_coverage(
agg["accounted"] += row.status != "unclassified"
unclassified = counts.pop("unclassified")
proposals = shape_ledger.proposal_summary(rows)
return {
"total": len(rows),
"accounted": len(rows) - unclassified,
"unclassified": unclassified,
"counts": counts,
# The proposer's standing (#2792): canon proposals awaiting a
# confirm, the largest derive-first groups, and what this refresh did.
"proposed": proposals["proposed"],
"derive_groups": proposals["derive_groups"],
"proposer": proposer_stats,
# Honesty flag, not decoration: every surface that shows the number
# is expected to carry it through.
"estimate": True,
@@ -438,6 +566,14 @@ def coverage_line(coverage: dict) -> str:
unclassified = coverage.get("unclassified", 0)
if unclassified:
line += f"; {unclassified} unclassified"
standing = []
if coverage.get("proposed"):
standing.append(f"{coverage['proposed']} proposed")
n_groups = len(coverage.get("derive_groups") or [])
if n_groups:
standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}")
if standing:
line += f" ({', '.join(standing)})"
gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
if gaps:
line += ", largest: " + ", ".join(gaps)
+467 -7
View File
@@ -21,9 +21,11 @@ an extracted shape.
"""
from __future__ import annotations
import difflib
import logging
import re
from datetime import datetime, timedelta, timezone
from typing import Iterable, NamedTuple
from sqlalchemy import select
@@ -65,15 +67,17 @@ def location_covers(loc_path: str, loc_symbol: str, path: str, name: str) -> boo
async def sync_repo_shapes(
project_id: int,
repo_key: str,
shapes: list[tuple[str, str, str]],
shapes: list,
*,
seen_marker: str,
) -> None:
"""Upsert one repo's extracted (path, kind, name) shapes into the ledger.
"""Upsert one repo's extracted shapes into the ledger.
``seen_marker`` is the commit the archive was read at when the forge can
say, else the ref name — provenance sugar; the row timestamps carry the
when.
``shapes`` are (path, kind, name) triples, or the richer ArchiveShape
records (#2792) whose 4th/5th fields — signature, body_sha — refresh the
row's content fingerprint. ``seen_marker`` is the commit the archive was
read at when the forge can say, else the ref name — provenance sugar;
the row timestamps carry the when.
"""
now = datetime.now(timezone.utc)
async with async_session() as session:
@@ -87,7 +91,10 @@ async def sync_repo_shapes(
).scalars().all()
by_key = {(r.path, r.symbol, r.kind): r for r in rows}
seen: set[tuple[str, str, str]] = set()
for path, kind, name in shapes:
for shape in shapes:
path, kind, name = shape[0], shape[1], shape[2]
signature = shape[3] if len(shape) > 3 else ""
body_sha = shape[4] if len(shape) > 4 else ""
key = (path, name, kind)
if key in seen:
continue
@@ -98,9 +105,14 @@ async def sync_repo_shapes(
project_id=project_id, repo_key=repo_key,
path=path, symbol=name, kind=kind,
first_seen_commit=seen_marker, last_seen_commit=seen_marker,
signature=signature, body_sha=body_sha,
))
continue
row.last_seen_commit = seen_marker
if signature:
row.signature = signature
if body_sha:
row.body_sha = body_sha
# A shape that vanished and came back is live again — the vanish
# stays visible in history via updated_at, not as a dead flag.
row.vanished_at = None
@@ -297,6 +309,10 @@ async def classify_shapes(
status = item["status"]
for row in matches:
row.status = status
# The machine proposes, judgment classifies: any judgment
# retires the standing proposal; a withdrawal also forgets
# the examination so the next refresh proposes afresh.
_clear_proposal(row, reexamine=(status == "unclassified"))
if status == "unclassified":
row.snippet_id = None
row.reason = None
@@ -324,12 +340,16 @@ async def list_project_shapes(
include_vanished: bool = False,
limit: int = 100,
offset: int = 0,
proposal: str = "",
) -> tuple[list[CodeShape], int]:
"""A filtered page of a project's ledger, with the unfiltered-match total.
([], 0) when the caller can't read the project — the same silence every
other project list gives. ``path`` matches the exact file or anything
beneath it, mirroring recorded-location semantics.
beneath it, mirroring recorded-location semantics. ``proposal`` narrows
to rows the proposer has spoken about: "any", "canon" (an instance-of-#N
suggestion), "derive" (a repeats-with-no-canon group), or one basis
name (symbol/reference/text/signature/semantic).
"""
from sqlalchemy import func, or_
@@ -349,6 +369,17 @@ async def list_project_shapes(
))
if snippet_id:
conds.append(CodeShape.snippet_id == snippet_id)
if proposal == "any":
conds.append(or_(
CodeShape.proposed_snippet_id.isnot(None),
CodeShape.proposal_group.isnot(None),
))
elif proposal == "canon":
conds.append(CodeShape.proposed_snippet_id.isnot(None))
elif proposal == "derive":
conds.append(CodeShape.proposal_group.isnot(None))
elif proposal:
conds.append(CodeShape.proposal_basis == proposal)
async with async_session() as session:
total = (
await session.execute(
@@ -589,6 +620,7 @@ async def stamp_write_path_instances(
row.reason = why
row.classified_by = "hook"
row.classified_at = now
_clear_proposal(row)
stamped.append({
"path": path, "symbol": name, "kind": kind,
"snippet_id": sid, "reason": why,
@@ -596,3 +628,431 @@ async def stamp_write_path_instances(
if stamped:
await session.commit()
return stamped
# --- the mechanical proposer (#2792): the machine proposes, judgment classifies
#
# Runs inside the coverage refresh — the one moment the shape BODIES exist
# (the archive is in memory; the ledger stores fingerprints, never code).
# Over every live unclassified row whose content changed since it was last
# examined, it tries, strongest first:
#
# symbol the shape bears a recorded canon's symbol at another location
# (a second definition of the canon's name — an instance or a
# duplicate to consolidate; either way, it answers to #N);
# text whitespace-insensitive containment either way between the
# body and the canon's recorded code (git-grep, in effect —
# a verbatim copy, or the canon's own call-site example);
# reference the body calls/uses a canon's symbol — the call-site shape
# the P7 backfill classified by hand (#2790);
# signature a sym whose definition line, name blanked, resembles the
# canon's (the move_*/resequence family shape);
# semantic the widest net: the body (concept-queried, as the write-path
# arm does) scores above the write-path threshold against a
# canon — capped per refresh, so the cost is bounded and a big
# ledger is worked through across refreshes.
#
# A hit is a PROPOSAL on the row (proposed_snippet_id/basis/score), never a
# classification: an agent confirms in batches (confirm_proposals) or judges
# otherwise (classify_shapes clears it). Rows with no canon hit are grouped
# by the derive-first rule (note 2786): the same body fingerprint in ≥2
# places, or the same name defined in ≥3 files, is "a repeating shape with
# no canon — derive one first", carried as proposal_basis="derive" + a group
# key so sessions see the consolidation candidates as one thing.
# Derive-first floors. Identical bodies twice is already a copy; a bare name
# needs more repetition before it reads as a family (setup/handler/register
# recur by convention, not by duplication).
_DERIVE_MIN_DUP = 2
_DERIVE_MIN_NAME = 3
# Semantic checks per repo per refresh — an embedding each (local fastembed),
# bounded so a 4,000-row ledger is worked through over refreshes, not in one.
_SEMANTIC_CAP = 150
# Signature resemblance floor, name blanked (difflib ratio) — and a length
# floor, because `def NAME():` resembles `def NAME(x):` at 0.95 while saying
# nothing; a family shape has parameters to resemble.
_SIGNATURE_FLOOR = 0.8
_SIGNATURE_MIN_LEN = 30
# Textual containment needs enough substance to mean anything.
_TEXT_FLOOR = 40
_BASIS_ORDER = ("symbol", "text", "reference", "signature")
class Canon(NamedTuple):
snippet_id: int
kind: str
symbol: str
locations: tuple[tuple[str, str], ...]
signature: str
code_norm: str
def _norm_text(text: str) -> str:
return " ".join((text or "").split())
def signature_similarity(sig_a: str, name_a: str, sig_b: str, name_b: str) -> float:
"""How alike two definition lines are once their own names are blanked —
`def move_event(project_id, event_id, after_id)` against
`def move_beat(project_id, beat_id, after_id)` reads high."""
a = _norm_text(sig_a)
b = _norm_text(sig_b)
if name_a:
a = a.replace(name_a, "NAME")
if name_b:
b = b.replace(name_b, "NAME")
if len(a) < _SIGNATURE_MIN_LEN or len(b) < _SIGNATURE_MIN_LEN:
return 0.0
return difflib.SequenceMatcher(None, a, b).ratio()
def text_contains(body: str, code: str) -> bool:
"""Whitespace-insensitive containment, either way, above a substance
floor — the proposer's git-grep."""
a = _norm_text(body)
b = _norm_text(code)
if len(a) < _TEXT_FLOOR or len(b) < _TEXT_FLOOR:
return False
return a in b or b in a
def match_canon(
kind: str, path: str, symbol: str, signature: str, body: str,
canons: Iterable[Canon],
) -> tuple[int, str, float] | None:
"""The strongest (snippet_id, basis, score) a shape earns against the
canon catalog, by the non-semantic bases — or None. Strongest by
basis order, then by score within the basis."""
best: dict[str, tuple[float, int]] = {}
def offer(basis: str, score: float, sid: int) -> None:
cur = best.get(basis)
if cur is None or score > cur[0]:
best[basis] = (score, sid)
norm_sym = _norm_symbol(symbol)
for c in canons:
if c.kind != kind:
continue
if c.symbol and _norm_symbol(c.symbol) == norm_sym:
if not any(location_covers(lp, ls, path, symbol) for lp, ls in c.locations):
offer("symbol", 1.0, c.snippet_id)
continue # its own location is canonical territory, not a proposal
if c.symbol and references_symbol(body, c.symbol, kind):
offer("reference", 0.9, c.snippet_id)
if c.code_norm and text_contains(body, c.code_norm):
offer("text", 0.95, c.snippet_id)
if kind == "sym" and c.signature:
ratio = signature_similarity(signature, symbol, c.signature, c.symbol)
if ratio >= _SIGNATURE_FLOOR:
offer("signature", round(ratio, 3), c.snippet_id)
for basis in _BASIS_ORDER:
if basis in best:
score, sid = best[basis]
return (sid, basis, score)
return None
async def canon_catalog(user_id: int) -> list[Canon]:
"""Every snippet this user can browse, as matchable canon — own projects
and shared ones alike, because family canon counts (note 2786)."""
from scribe.models.note import Note
from scribe.services.access import browsable_notes_clause
from scribe.services.coverage import extract_definitions
from scribe.services.snippets import SNIPPET_NOTE_TYPE, snippet_fields
async with async_session() as session:
notes = (
await session.execute(
select(Note).where(
browsable_notes_clause(user_id),
Note.note_type == SNIPPET_NOTE_TYPE,
Note.deleted_at.is_(None),
)
)
).scalars().all()
out: list[Canon] = []
for note in notes:
fields = snippet_fields(note)
symbol = (fields.get("symbol") or "").strip()
kind = snippet_kind(symbol, fields.get("language") or "")
code = fields.get("code") or ""
defs = extract_definitions(code)
signature = ""
if defs:
own = next(
(d for d in defs if _norm_symbol(d.name) == _norm_symbol(symbol)), defs[0]
)
signature = own.signature
out.append(Canon(
int(note.id), kind, symbol,
tuple(
((loc.get("path") or ""), (loc.get("symbol") or ""))
for loc in fields.get("locations") or []
),
signature, _norm_text(code),
))
return out
def _clear_proposal(row: CodeShape, *, reexamine: bool = False) -> None:
row.proposed_snippet_id = None
row.proposal_basis = None
row.proposal_score = None
row.proposal_group = None
if reexamine:
row.proposed_at = None
row.proposed_sha = ""
def _substance(text: str) -> int:
return len("".join((text or "").split()))
async def _semantic_canon(
user_id: int, body: str, allowed: set[int]
) -> tuple[int, float] | None:
from scribe.services.embeddings import semantic_search_notes
from scribe.services.plugin_context import (
WRITEPATH_DEFAULT_THRESHOLD, WRITEPATH_MIN_CODE_CHARS, concept_query,
)
if _substance(body) < WRITEPATH_MIN_CODE_CHARS or not allowed:
return None
query = concept_query(body) or body
hits = await semantic_search_notes(
user_id, query, limit=3, threshold=WRITEPATH_DEFAULT_THRESHOLD,
note_type="snippet", scope="browse",
)
for score, note in hits:
if int(note.id) in allowed:
return int(note.id), round(float(score), 3)
return None
async def propose_for_repo(
user_id: int,
project_id: int,
repo_key: str,
definitions: list,
*,
canons: list[Canon] | None = None,
semantic_cap: int = _SEMANTIC_CAP,
) -> dict:
"""Examine one repo's live unclassified rows against canon and record
proposals. ``definitions`` are the ArchiveShape records the sync just
upserted (their bodies are the matching material). Rows whose fingerprint
is unchanged since their last examination are skipped; rows that only
the capped semantic pass could not reach stay unexamined, so the next
refresh reaches the next slice. Returns counts."""
if canons is None:
canons = await canon_catalog(user_id)
by_key = {(d[0], d[1], d[2]): d for d in definitions}
sym_canon_ids = {c.snippet_id for c in canons if c.kind == "sym"}
now = datetime.now(timezone.utc)
examined = proposed = checked = 0
async with async_session() as session:
rows = (
await session.execute(
select(CodeShape).where(
CodeShape.project_id == project_id,
CodeShape.repo_key == repo_key,
CodeShape.status == "unclassified",
CodeShape.vanished_at.is_(None),
)
)
).scalars().all()
semantic_todo: list[tuple[CodeShape, object]] = []
for row in rows:
d = by_key.get((row.path, row.kind, row.symbol))
if d is None:
continue
signature, body_sha, body = d[3], d[4], d[5]
if row.proposed_at is not None and row.proposed_sha == body_sha:
continue
examined += 1
group = row.proposal_group # derive grouping is reassigned below
hit = match_canon(row.kind, row.path, row.symbol, signature, body, canons)
_clear_proposal(row)
row.proposal_group = group
row.proposed_at = now
row.proposed_sha = body_sha
if hit:
row.proposed_snippet_id, row.proposal_basis, row.proposal_score = hit
row.proposal_group = None
proposed += 1
elif row.kind == "sym":
semantic_todo.append((row, d))
for i, (row, d) in enumerate(semantic_todo):
if i >= semantic_cap:
# Not reached this refresh: leave it unexamined so the next
# refresh picks it up, rather than stamping a false "nothing".
row.proposed_at = None
row.proposed_sha = ""
continue
checked += 1
try:
found = await _semantic_canon(user_id, d[5], sym_canon_ids)
except Exception:
logger.warning("semantic proposal failed", exc_info=True)
found = None
if found:
row.proposed_snippet_id, row.proposal_score = found
row.proposal_basis = "semantic"
row.proposal_group = None
proposed += 1
await session.commit()
return {"examined": examined, "proposed": proposed, "semantic_checked": checked}
def derive_groups(
rows: Iterable[tuple[str, str, str, str]]
) -> dict[tuple[str, str, str], str]:
"""The derive-first grouping over (path, kind, symbol, body_sha) rows
that matched no canon: {(path, kind, symbol): group_key}. Identical
bodies in ≥2 places group as `dup:<sha>`; the same name defined in ≥3
files groups as `name:<kind>:<symbol>`; a row joins at most one group,
the copy before the name."""
by_sha: dict[str, list[tuple[str, str, str]]] = {}
by_name: dict[tuple[str, str], list[tuple[str, str, str]]] = {}
for path, kind, symbol, sha in rows:
key = (path, kind, symbol)
if sha:
by_sha.setdefault(sha, []).append(key)
by_name.setdefault((kind, _norm_symbol(symbol)), []).append(key)
out: dict[tuple[str, str, str], str] = {}
for sha, keys in by_sha.items():
if len(set(keys)) >= _DERIVE_MIN_DUP:
for key in keys:
out.setdefault(key, f"dup:{sha}")
for (kind, symbol), keys in by_name.items():
if len({k[0] for k in keys}) >= _DERIVE_MIN_NAME:
for key in keys:
out.setdefault(key, f"name:{kind}:{symbol}")
return out
async def apply_derive_groups(project_id: int) -> int:
"""Recompute derive-first groups over the project's live unclassified
rows that carry no canon proposal; returns how many rows are grouped."""
now = datetime.now(timezone.utc)
grouped = 0
async with async_session() as session:
rows = (
await session.execute(
select(CodeShape).where(
CodeShape.project_id == project_id,
CodeShape.status == "unclassified",
CodeShape.vanished_at.is_(None),
CodeShape.proposed_snippet_id.is_(None),
)
)
).scalars().all()
groups = derive_groups(
(r.path, r.kind, r.symbol, r.body_sha or "") for r in rows
)
sizes: dict[str, int] = {}
for g in groups.values():
sizes[g] = sizes.get(g, 0) + 1
for row in rows:
key = groups.get((row.path, row.kind, row.symbol))
if key:
row.proposal_basis = "derive"
row.proposal_group = key
row.proposal_score = float(sizes[key])
if row.proposed_at is None:
row.proposed_at = now
grouped += 1
elif row.proposal_group:
row.proposal_basis = None
row.proposal_group = None
row.proposal_score = None
await session.commit()
return grouped
def proposal_summary(rows: Iterable[CodeShape], *, top: int = 8) -> dict:
"""The readout's view of the proposer's standing: how many canon
proposals await confirmation, and the largest derive-first groups."""
proposed = 0
groups: dict[str, dict] = {}
for row in rows:
if row.status != "unclassified":
continue
if row.proposed_snippet_id is not None:
proposed += 1
elif row.proposal_group:
g = groups.setdefault(row.proposal_group, {
"group": row.proposal_group, "kind": row.kind,
"label": (
("." if row.kind == "css" else "") + row.symbol
if row.proposal_group.startswith("name:")
else f"{row.symbol} (identical body)"
),
"size": 0, "paths": [],
})
g["size"] += 1
if len(g["paths"]) < 3:
g["paths"].append(row.path)
ranked = sorted(groups.values(), key=lambda g: (-g["size"], g["group"]))
return {"proposed": proposed, "derive_groups": ranked[:top]}
async def confirm_proposals(
user_id: int,
project_id: int,
*,
snippet_id: int = 0,
path: str = "",
basis: str = "",
min_score: float = 0.0,
) -> dict:
"""Turn reviewed canon proposals into `instance` rows, in one batch.
At least one of snippet_id / path / basis must narrow the batch — "confirm
everything proposed" without having looked is not a judgment. Each row
becomes instance-of-its-proposed-snippet, classified_by="agent", reason
naming the basis and score; the proposal is retired. Returns
{"confirmed": N}."""
from scribe.services import access
if not (snippet_id or path.strip() or basis.strip()):
raise ValueError(
"name what you reviewed: confirm by snippet_id, path, and/or basis"
)
if not await access.can_write_project(user_id, project_id):
raise ValueError(f"project {project_id} not found or no write access")
from sqlalchemy import or_
conds = [
CodeShape.project_id == project_id,
CodeShape.status == "unclassified",
CodeShape.vanished_at.is_(None),
CodeShape.proposed_snippet_id.isnot(None),
]
if snippet_id:
conds.append(CodeShape.proposed_snippet_id == snippet_id)
if path.strip():
clean = path.strip().strip("/")
conds.append(or_(CodeShape.path == clean, CodeShape.path.like(clean + "/%")))
if basis.strip():
conds.append(CodeShape.proposal_basis == basis.strip())
now = datetime.now(timezone.utc)
confirmed = 0
async with async_session() as session:
rows = (await session.execute(select(CodeShape).where(*conds))).scalars().all()
for row in rows:
if (row.proposal_score or 0.0) < min_score:
continue
row.status = "instance"
row.snippet_id = row.proposed_snippet_id
row.reason = (
f"confirmed {row.proposal_basis} proposal"
f" ({(row.proposal_score or 0.0):.2f})"
)
row.classified_by = "agent"
row.classified_at = now
_clear_proposal(row)
confirmed += 1
await session.commit()
return {"confirmed": confirmed}
+142
View File
@@ -351,3 +351,145 @@ async def test_recent_pulls_reads_the_usage_stream(seeded):
assert sid in pulls
assert sid + 1000 not in pulls
assert sid + 2000 not in pulls
# --- #2792: the mechanical proposer against real rows -------------------------
def _quiet_semantic():
"""The semantic basis needs the embedder; these tests prove the other
bases and the bookkeeping, so it answers "nothing" here."""
from unittest.mock import AsyncMock, patch
from scribe.services import shape_ledger
return patch.object(shape_ledger, "_semantic_canon", AsyncMock(return_value=None))
def _defs(*items):
"""ArchiveShape-like records: (path, kind, name, signature, body_sha, body)."""
import hashlib
out = []
for path, kind, name, signature, body in items:
sha = hashlib.sha1(" ".join(body.split()).encode()).hexdigest()[:16]
out.append((path, kind, name, signature, sha, body))
return out
@pytest.mark.integration
async def test_proposer_proposes_and_confirm_classifies(seeded):
"""Bodies in hand, the proposer records proposals on unclassified rows —
symbol (a second `factory` elsewhere), reference (a call site), and
nothing for the unrelated — skips rows whose content it already judged,
and a scoped confirm turns proposals into agent instances while a
classify on another retires its proposal."""
from scribe.services.shape_ledger import (
confirm_proposals, propose_for_repo,
)
owner, other, pid, sid = (
seeded["owner"], seeded["other"], seeded["pid"], seeded["snippet"]
)
defs = _defs(
("src/app.py", "sym", "make_app", "def make_app():", "def make_app():\n app = factory()\n return app"),
("src/app.py", "sym", "Config", "class Config:", "class Config:\n debug = False"),
("src/util.py", "sym", "helper", "def helper(x):", "def helper(x):\n return x"),
("src/dup.py", "sym", "factory", "def factory():", "def factory():\n return 1"),
("web/button.css", "css", "btn", ".btn {", ".btn {\n color: red;\n}"),
)
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
with _quiet_semantic():
stats = await propose_for_repo(owner, pid, REPO, defs)
assert stats == {"examined": 5, "proposed": 2, "semantic_checked": 2}
rows, total = await list_project_shapes(owner, pid, proposal="canon")
by_symbol = {r.symbol: r for r in rows}
assert total == 2
assert by_symbol["factory"].proposal == {"basis": "symbol", "score": 1.0, "snippet_id": sid}
assert by_symbol["make_app"].proposal == {"basis": "reference", "score": 0.9, "snippet_id": sid}
rows, _ = await list_project_shapes(owner, pid, proposal="reference")
assert [r.symbol for r in rows] == ["make_app"]
# Same content again → nothing re-examined (the semantic cap would
# otherwise be spent on the same rows every refresh). A cap that leaves
# rows unreached leaves them UNexamined, so the next refresh gets them.
with _quiet_semantic():
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 0
await classify_shapes(owner, pid, [
{"path": "src/util.py", "symbol": "helper", "status": "unclassified"},
])
assert (await propose_for_repo(owner, pid, REPO, defs, semantic_cap=0))["semantic_checked"] == 0
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 1
# Outsider can't confirm; the owner confirms by snippet, scoped.
with pytest.raises(ValueError):
await confirm_proposals(other, pid, snippet_id=sid)
assert await confirm_proposals(owner, pid, basis="symbol") == {"confirmed": 1}
rows, _ = await list_project_shapes(owner, pid, snippet_id=sid)
factory = next(r for r in rows if r.symbol == "factory")
assert factory.status == "instance" and factory.classified_by == "agent"
assert factory.reason == "confirmed symbol proposal (1.00)"
assert factory.proposal is None
# A judgment on a proposed row retires the proposal; withdrawing a
# judgment forgets the examination so the next pass proposes afresh.
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "exempt", "reason": "bootstrap"},
])
rows, _ = await list_project_shapes(owner, pid, path="src/app.py")
make_app = next(r for r in rows if r.symbol == "make_app")
assert make_app.status == "exempt" and make_app.proposal is None
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "unclassified"},
])
with _quiet_semantic():
assert (await propose_for_repo(owner, pid, REPO, defs))["proposed"] == 1
rows, _ = await list_project_shapes(owner, pid, proposal="canon")
assert [r.symbol for r in rows] == ["make_app"]
@pytest.mark.integration
async def test_derive_groups_land_on_rows_and_in_the_summary(seeded):
"""Shapes with no canon hit that repeat — identical bodies in two files,
the same name in three — carry a derive proposal, and the readout ranks
the families. A canon proposal keeps a row out of any derive group."""
from scribe.services.shape_ledger import (
apply_derive_groups, live_rows, propose_for_repo, proposal_summary,
)
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
defs = _defs(
("a/one.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
("a/two.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
("b/x.css", "css", "card", ".card {", ".card { padding: 1px }"),
("b/y.css", "css", "card", ".card {", ".card { padding: 2px }"),
("b/z.css", "css", "card", ".card {", ".card { padding: 3px }"),
("c/only.py", "sym", "alone", "def alone():", "def alone():\n return 0"),
("c/use.py", "sym", "boot", "def boot():", "def boot():\n return factory()"),
)
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
with _quiet_semantic():
await propose_for_repo(owner, pid, REPO, defs)
assert await apply_derive_groups(pid) == 5
rows, total = await list_project_shapes(owner, pid, proposal="derive")
assert total == 5
groups = {(r.path, r.symbol): r.proposal for r in rows}
assert groups[("a/one.py", "slug")]["group"] == groups[("a/two.py", "slug")]["group"]
assert groups[("a/one.py", "slug")]["group"].startswith("dup:")
assert groups[("b/x.css", "card")] == {"basis": "derive", "score": 3.0, "group": "name:css:card"}
rows, _ = await list_project_shapes(owner, pid, proposal="any")
assert {r.symbol for r in rows} == {"slug", "card", "boot"} # boot: reference proposal
summary = proposal_summary(await live_rows(pid))
assert summary["proposed"] == 1
assert [g["group"] for g in summary["derive_groups"]][0] == "name:css:card"
assert summary["derive_groups"][0]["label"] == ".card"
assert summary["derive_groups"][0]["size"] == 3
# One of the css copies gets judged → the group shrinks on the next pass.
await classify_shapes(owner, pid, [
{"path": "b/z.css", "symbol": "card", "status": "exempt", "reason": "print sheet"},
])
await apply_derive_groups(pid)
rows, _ = await list_project_shapes(owner, pid, proposal="derive")
assert {r.symbol for r in rows} == {"slug"} # 2 files < the name floor
+51
View File
@@ -456,3 +456,54 @@ async def test_unservable_binding_measures_nothing(seeded):
await set_binding(uid, "https://github.com/somebody/else.git", other_pid)
assert await compute_coverage(uid, other_pid, selector=_selector(_tarball(TREE))) is None
# --- #2792: fingerprints and the proposer's readout --------------------------
def test_extract_definitions_fingerprints_each_block():
"""The block rule across the language families the extractor knows: a
Python def ends at the next top-level statement, a braces/CSS block keeps
its closer, and comments/decorators don't move the hash."""
from scribe.services.coverage import extract_definitions
text = (
"import os\n\n"
"def a(x):\n # comment\n return x + 1\n\n\n"
"class B:\n def m(self):\n return 2\n\n"
".btn {\n color: red;\n}\n"
"export const f = (x) => {\n return x;\n};\n"
)
defs = {d.name: d for d in extract_definitions(text)}
assert set(defs) == {"a", "B", "m", "btn", "f"}
assert defs["a"].signature == "def a(x):"
assert defs["a"].body.startswith("def a(x):\n # comment\n return x + 1")
assert "class B" not in defs["a"].body
assert defs["B"].body.rstrip().endswith("return 2")
assert defs["btn"].body == ".btn {\n color: red;\n}"
assert defs["f"].body == "export const f = (x) => {\n return x;\n};"
assert all(len(d.body_sha) == 16 for d in defs.values())
# Comment changes don't change what the shape IS; code changes do.
again = {d.name: d for d in extract_definitions(text.replace("# comment", "# other"))}
assert again["a"].body_sha == defs["a"].body_sha
changed = {d.name: d for d in extract_definitions(text.replace("x + 1", "x + 2"))}
assert changed["a"].body_sha != defs["a"].body_sha
# And the identity view is unchanged for the hook mirror.
from scribe.services.coverage import extract_shapes
assert extract_shapes(text) == [(d.kind, d.name) for d in extract_definitions(text)]
def test_coverage_line_names_the_proposers_standing():
from scribe.services.coverage import coverage_line
base = {
"total": 100, "accounted": 10, "unclassified": 90,
"counts": {"canonical": 10, "instance": 0, "variant": 0, "exempt": 0},
"computed_at": "2026-08-21T00:00:00+00:00",
"largest_gaps": [{"dir": "src", "unclassified": 90, "total": 90}],
}
assert coverage_line(base).endswith("; 90 unclassified, largest: src")
line = coverage_line({**base, "proposed": 40, "derive_groups": [{"group": "a"}, {"group": "b"}]})
assert "; 90 unclassified (40 proposed, 2 derive groups), largest: src" in line
line = coverage_line({**base, "proposed": 0, "derive_groups": [{"group": "a"}]})
assert "(1 derive group)" in line
+141
View File
@@ -5,6 +5,8 @@ token-free serialisation. The sync pass (step 2) and the classification
surface (step 3) grow their tests here; DB-backed behavior lands in the
integration lane once there is behavior to exercise.
"""
import pytest
from scribe.models import Base
from scribe.models.code_shape import SHAPE_CLASSIFIERS, SHAPE_STATUSES, CodeShape
@@ -135,3 +137,142 @@ def test_hook_is_a_server_internal_classifier():
assert "hook" in SHAPE_CLASSIFIERS
assert "hook" not in _CALLER_VIAS
# --- step 6: the mechanical proposer (pure) ---------------------------------
def test_proposal_columns_and_vocabulary_are_pinned():
"""Fingerprints + the proposer's standing suggestion live on the row; the
basis vocabulary is fixed, with `derive` the odd one out (a group, not a
snippet)."""
from scribe.models.code_shape import PROPOSAL_BASES
cols = CodeShape.__table__.c
for name in ("signature", "body_sha", "proposed_snippet_id", "proposal_basis",
"proposal_score", "proposal_group", "proposed_at", "proposed_sha"):
assert name in cols, name
fk = next(iter(cols.proposed_snippet_id.foreign_keys))
assert fk.ondelete == "SET NULL" and fk.column.table.name == "notes"
assert "ix_code_shapes_proposed" in {ix.name for ix in CodeShape.__table__.indexes}
assert PROPOSAL_BASES == ("symbol", "text", "reference", "signature", "semantic", "derive")
def test_row_proposal_property_is_one_object_or_none():
row = CodeShape(project_id=1, repo_key="r", path="a.py", symbol="f", kind="sym")
assert row.proposal is None
row.proposed_snippet_id, row.proposal_basis, row.proposal_score = 9, "symbol", 1.0
assert row.proposal == {"basis": "symbol", "score": 1.0, "snippet_id": 9}
row.proposed_snippet_id = None
row.proposal_basis, row.proposal_group, row.proposal_score = "derive", "dup:abc", 3.0
assert row.proposal == {"basis": "derive", "score": 3.0, "group": "dup:abc"}
def test_signature_similarity_blanks_the_names():
from scribe.services.shape_ledger import signature_similarity as sim
a = "def move_event(project_id: int, event_id: int, after_id: int | None):"
b = "def move_beat(project_id: int, beat_id: int, after_id: int | None):"
assert sim(a, "move_event", b, "move_beat") > 0.85
assert sim(a, "move_event", "def export_pdf(manuscript, design, fonts):", "export_pdf") < 0.6
assert sim("", "x", b, "move_beat") == 0.0
# Trivial signatures resemble everything and mean nothing — floored out.
assert sim("def helper(x):", "helper", "def make_app():", "make_app") == 0.0
def test_text_containment_is_whitespace_insensitive_with_a_floor():
from scribe.services.shape_ledger import text_contains
code = "const ok = await confirmed({ title: 'Delete?', confirmLabel: 'Delete' });\nif (!ok) return;"
body = "async function onDelete() {\n const ok = await confirmed({\n title: 'Delete?',\n confirmLabel: 'Delete'\n });\n if (!ok) return;\n}"
assert text_contains(body, code)
assert text_contains(code, body)
assert not text_contains("x = 1", "x = 1") # below the substance floor
def _canon(sid, kind="sym", symbol="", locations=(), signature="", code=""):
from scribe.services.shape_ledger import Canon, _norm_text
return Canon(sid, kind, symbol, tuple(locations), signature, _norm_text(code))
def test_match_canon_orders_bases_strongest_first_and_respects_kind():
from scribe.services.shape_ledger import match_canon
confirmed = _canon(
7, "sym", "confirmed", [("frontend/src/composables/useConfirm.ts", "confirmed")],
"export async function confirmed(opts: ConfirmOptions): Promise<boolean> {",
"export async function confirmed(opts: ConfirmOptions): Promise<boolean> { /* singleton */ }",
)
mover = _canon(
8, "sym", "move_beat", [("src/forge/plot.py", "move_beat")],
"def move_beat(project_id: int, beat_id: int, after_id: int | None) -> None:",
)
btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")],
".btn-primary {", ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }")
canons = [confirmed, mover, btn]
# symbol: a second `confirmed` defined elsewhere answers to #7 — but the
# canon's own location never does (that row is canonical, not a proposal).
assert match_canon("sym", "src/other.ts", "confirmed", "function confirmed() {", "", canons) == (7, "symbol", 1.0)
assert match_canon("sym", "frontend/src/composables/useConfirm.ts", "confirmed",
"export async function confirmed(", "", canons) is None
# reference: a call site of the canon.
body = "async function onTrash() {\n const ok = await confirmed({ title: 'x' });\n if (!ok) return;\n}"
assert match_canon("sym", "c.vue", "onTrash", "async function onTrash() {", body, canons) == (7, "reference", 0.9)
# signature: the family shape, names blanked.
hit = match_canon("sym", "src/forge/timeline.py", "move_event",
"def move_event(project_id: int, event_id: int, after_id: int | None) -> None:",
" pass", canons)
assert hit and hit[0] == 8 and hit[1] == "signature" and hit[2] >= 0.8
# text: the canon's code contains the shape's body (a css copy), kind-matched —
# the same text as a `sym` shape matches no css canon.
css_body = ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }"
assert match_canon("css", "web/other.css", "btn-big", ".btn-big {", css_body, canons) == (9, "text", 0.95)
assert match_canon("sym", "web/other.css", "btn-big", ".btn-big {", css_body, canons) is None
# nothing in play
assert match_canon("sym", "x.py", "unrelated", "def unrelated(a, b, c, d, e):", "return 1", canons) is None
def test_match_canon_symbol_beats_everything_including_css_copies():
"""The previous test's css `btn-primary`-elsewhere case, stated plainly:
a second definition of the canon's own name is the symbol basis."""
from scribe.services.shape_ledger import match_canon
btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")], ".btn-primary {", ".btn-primary { color: red; padding: 4px 8px; border-radius: 4px; }")
assert match_canon("css", "web/other.css", "btn-primary", ".btn-primary {", ".btn-primary { color: blue }", [btn]) == (9, "symbol", 1.0)
def test_derive_groups_copy_before_name_with_floors():
from scribe.services.shape_ledger import derive_groups
rows = [
("a.py", "sym", "helper", "sha1"), ("b.py", "sym", "helper", "sha1"), # identical copies
("c.py", "sym", "helper", "sha9"), # same name, 3rd file
("d.css", "css", "btn", "s1"), ("e.css", "css", "btn", "s2"), ("f.css", "css", "btn", "s3"),
("g.py", "sym", "main", "s4"), ("h.py", "sym", "main", "s5"), # only 2 files → no name group
("i.py", "sym", "one", "s6"),
]
g = derive_groups(rows)
assert g[("a.py", "sym", "helper")] == "dup:sha1" == g[("b.py", "sym", "helper")]
assert g[("c.py", "sym", "helper")] == "name:sym:helper"
assert g[("d.css", "css", "btn")] == "name:css:btn"
assert ("g.py", "sym", "main") not in g
assert ("i.py", "sym", "one") not in g
def test_confirm_requires_a_named_scope():
import asyncio
from scribe.services.shape_ledger import confirm_proposals
with pytest.raises(ValueError) as err:
asyncio.run(confirm_proposals(1, 2))
assert "name what you reviewed" in str(err.value)
def test_proposer_tools_are_mounted():
from scribe.mcp.server import build_mcp_server
mcp = build_mcp_server()
assert mcp._tool_manager.get_tool("confirm_shape_proposals") is not None
tool = mcp._tool_manager.get_tool("list_shapes")
assert "proposal" in tool.parameters.get("properties", {})