feat(ledger): mechanical proposer — every refresh proposes instances against canon and groups derive-first candidates; agents confirm in batches (#2792, milestone 294 step 6)
CI & Build / Python lint (push) Successful in 5s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Failing after 34s
CI & Build / TypeScript typecheck (push) Successful in 35s
CI & Build / Python tests (push) Successful in 1m17s
CI & Build / Build & push image (push) Successful in 48s
CI & Build / Python lint (push) Successful in 5s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Failing after 34s
CI & Build / TypeScript typecheck (push) Successful in 35s
CI & Build / Python tests (push) Successful in 1m17s
CI & Build / Build & push image (push) Successful in 48s
Shapes now carry a content fingerprint (signature + whitespace/comment- insensitive body_sha; migration 0080) and the proposer runs inside the coverage refresh, the one moment bodies exist: symbol elsewhere → textual containment → body references the canon → signature resemblance → semantic (capped per refresh, unreached rows stay unexamined for the next). A hit is a proposal on the row (proposed_snippet_id/basis/score), never a classification; rows with no canon hit group by the derive-first rule (identical body in ≥2 places, same name in ≥3 files) as proposal_basis= derive + a group key. list_shapes(proposal=any|canon|derive|<basis>) is the queue; confirm_shape_proposals(project_id, snippet_id|path|basis) confirms in batches as agent instances; any classify_shapes/hook stamp retires the proposal. Readout carries proposed + derive_groups (line, payload, card). Plugin 0.1.35 (skill: the machine proposes, judgment classifies). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
"""Shape fingerprints + the mechanical proposer's columns (#2792, milestone 294)
|
||||
|
||||
Revision ID: 0080
|
||||
Revises: 0079
|
||||
Create Date: 2026-08-21
|
||||
|
||||
Two additions to the ledger. `signature` / `body_sha` fingerprint each shape
|
||||
(definition line + a whitespace/comment-insensitive hash of its block) so the
|
||||
proposer can match on content and a later drift recheck can notice change,
|
||||
without the ledger ever storing code. The proposal columns carry the
|
||||
proposer's standing suggestion for an unclassified row — instance-of-#N with
|
||||
a basis and score, or a derive-first group key — and `proposed_sha`
|
||||
remembers the content it was judged at so a refresh re-examines only what
|
||||
changed. Mechanical and recomputable: a restore that lacks them loses
|
||||
nothing the next refresh does not rebuild.
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0080"
|
||||
down_revision = "0079"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("code_shapes", sa.Column("signature", sa.Text(), nullable=False, server_default=""))
|
||||
op.add_column("code_shapes", sa.Column("body_sha", sa.Text(), nullable=False, server_default=""))
|
||||
op.add_column(
|
||||
"code_shapes",
|
||||
sa.Column(
|
||||
"proposed_snippet_id",
|
||||
sa.BigInteger(),
|
||||
sa.ForeignKey("notes.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
op.add_column("code_shapes", sa.Column("proposal_basis", sa.Text(), nullable=True))
|
||||
op.add_column("code_shapes", sa.Column("proposal_score", sa.Float(), nullable=True))
|
||||
op.add_column("code_shapes", sa.Column("proposal_group", sa.Text(), nullable=True))
|
||||
op.add_column("code_shapes", sa.Column("proposed_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column("code_shapes", sa.Column("proposed_sha", sa.Text(), nullable=False, server_default=""))
|
||||
op.create_index(
|
||||
"ix_code_shapes_proposed", "code_shapes", ["project_id", "proposed_snippet_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_code_shapes_proposed", table_name="code_shapes")
|
||||
for col in (
|
||||
"proposed_sha", "proposed_at", "proposal_group", "proposal_score",
|
||||
"proposal_basis", "proposed_snippet_id", "body_sha", "signature",
|
||||
):
|
||||
op.drop_column("code_shapes", col)
|
||||
@@ -427,6 +427,13 @@ interface CoverageGap {
|
||||
unclassified: number;
|
||||
total: number;
|
||||
}
|
||||
interface DeriveGroup {
|
||||
group: string;
|
||||
kind: string;
|
||||
label: string;
|
||||
size: number;
|
||||
paths: string[];
|
||||
}
|
||||
interface Coverage {
|
||||
total: number;
|
||||
accounted: number;
|
||||
@@ -436,6 +443,10 @@ interface Coverage {
|
||||
computed_at: string;
|
||||
repos: { repo: string; ref: string; total: number; accounted: number }[];
|
||||
largest_gaps: CoverageGap[];
|
||||
// The mechanical proposer's standing (#2792): canon proposals awaiting an
|
||||
// agent's confirm, and the biggest repeats-with-no-canon families.
|
||||
proposed?: number;
|
||||
derive_groups?: DeriveGroup[];
|
||||
}
|
||||
|
||||
const coverage = ref<Coverage | null>(null);
|
||||
@@ -746,6 +757,24 @@ async function confirmDelete() {
|
||||
{{ gap.dir }} <span class="coverage-gap-count">{{ gap.unclassified }}</span>
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
v-if="coverage.proposed || coverage.derive_groups?.length"
|
||||
class="coverage-gaps"
|
||||
title="The proposer matched these against canon; an agent confirms them in batches (confirm_shape_proposals)."
|
||||
>
|
||||
<span class="coverage-gaps-label">Proposed:</span>
|
||||
<span v-if="coverage.proposed" class="coverage-gap-chip">
|
||||
{{ coverage.proposed }} awaiting confirm
|
||||
</span>
|
||||
<span
|
||||
v-for="g in coverage.derive_groups || []"
|
||||
:key="g.group"
|
||||
class="coverage-gap-chip"
|
||||
:title="'Repeats with no canon — derive one first. ' + g.paths.join(', ')"
|
||||
>
|
||||
{{ g.label }} <span class="coverage-gap-count">×{{ g.size }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
<p v-else class="coverage-empty">
|
||||
Not measured yet — Refresh reads the bound repo's definitions into
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "scribe",
|
||||
"description": "Scribe system-of-record for Claude Code: MCP tools over your notes/tasks/projects/rules, a session-start push channel that surfaces your always-on rules + active-project context, process-skills (writing-plans, systematic-debugging, verification, brainstorming, reusing-code), and your saved Scribe Processes auto-surfaced as skills (/scribe:sync). Replaces superpowers + file-memory with one app-backed plugin.",
|
||||
"version": "0.1.34",
|
||||
"version": "0.1.35",
|
||||
"author": { "name": "Bryan Van Deusen" },
|
||||
"mcpServers": {
|
||||
"scribe": {
|
||||
|
||||
@@ -51,6 +51,27 @@ need a hand judgment:
|
||||
judgment: it never overrides a classification you made, and a
|
||||
`classify_shapes` call overrides it.
|
||||
|
||||
## The machine proposes, judgment classifies
|
||||
|
||||
Every coverage refresh runs the **mechanical proposer** over the unclassified
|
||||
rows: same symbol as a canon elsewhere → textual containment → body
|
||||
references a canon → signature resemblance → semantic (capped per refresh). A hit
|
||||
is a *proposal* on the row, never a classification. Work the queue in bulk:
|
||||
|
||||
1. `list_shapes(project_id, proposal="canon", snippet_id=N)` or
|
||||
`path="dir"` — read the page; `proposal` carries snippet_id, basis, score.
|
||||
2. `confirm_shape_proposals(project_id, snippet_id=N)` (or `path=`,
|
||||
`basis=`) for the ones that hold — hundreds at a time; `symbol` and
|
||||
`reference` proposals are near-certain, `semantic` deserves a look.
|
||||
3. `classify_shapes` the rest — variant, exempt, or instance of a different
|
||||
snippet. Any judgment retires the proposal.
|
||||
|
||||
`list_shapes(project_id, proposal="derive")` lists the **derive-first
|
||||
candidates** — the same body in ≥2 places or the same name defined in ≥3
|
||||
files, with no canon at all (`proposal.group` names the family; the coverage
|
||||
payload's `derive_groups` ranks the biggest). That is the consolidation
|
||||
queue, not a classification queue: see below.
|
||||
|
||||
## The derive-first rule
|
||||
|
||||
N same-shaped occurrences matching **no** recorded canon is never N loose
|
||||
|
||||
@@ -62,6 +62,7 @@ async def list_shapes(
|
||||
include_vanished: bool = False,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
proposal: str = "",
|
||||
) -> dict:
|
||||
"""Read a project's shape ledger — `status="unclassified"` IS the todo.
|
||||
|
||||
@@ -75,6 +76,11 @@ async def list_shapes(
|
||||
snippet_id: rows classified against this snippet — a consumer map.
|
||||
include_vanished: include shapes no longer in the tree (history).
|
||||
limit/offset: page through big ledgers (limit caps at 500).
|
||||
proposal: the proposer's queue (#2792) — "any", "canon" (rows the
|
||||
machine thinks are an instance of a snippet: `proposal` carries
|
||||
snippet_id, basis, score), "derive" (rows that repeat with NO
|
||||
canon: `proposal.group` names the family), or one basis
|
||||
(symbol/text/reference/signature/semantic).
|
||||
|
||||
Returns {"shapes": [...], "total": N} — total counts every match, not
|
||||
just this page. Each row's `classified_by` says who judged: agent /
|
||||
@@ -84,20 +90,61 @@ async def list_shapes(
|
||||
was stamped an instance with the evidence in `reason`. A hook row is
|
||||
overridable by any classify_shapes call; it never overrides yours.
|
||||
|
||||
Classify what you can judge with classify_shapes; a repeating shape
|
||||
with NO recorded canon is a derive-one-first moment (consolidate onto a
|
||||
reference, create_snippet it, then classify the rest against it), never
|
||||
N loose classifications.
|
||||
THE FAST PATH through a big todo is the proposer's queue: every coverage
|
||||
refresh matches unclassified shapes against canon (strongest basis
|
||||
first: same symbol elsewhere → textual containment → body references
|
||||
the canon → signature resemblance → semantic) and attaches a
|
||||
`proposal` to each row it can speak for. Review `proposal="canon"` by
|
||||
snippet or directory, then confirm_shape_proposals the ones that hold —
|
||||
hundreds at a time — and classify_shapes the rest (variant/exempt, or
|
||||
instance of a different snippet). `proposal="derive"` lists the
|
||||
derive-first candidates: a repeating shape with NO recorded canon is
|
||||
never N loose classifications — consolidate onto a reference,
|
||||
create_snippet it, then classify the group against it.
|
||||
"""
|
||||
uid = current_user_id()
|
||||
rows, total = await shape_ledger_svc.list_project_shapes(
|
||||
uid, project_id,
|
||||
status=status, path=path, snippet_id=snippet_id,
|
||||
include_vanished=include_vanished, limit=limit, offset=offset,
|
||||
proposal=proposal,
|
||||
)
|
||||
return {"shapes": [r.to_dict() for r in rows], "total": total}
|
||||
|
||||
|
||||
async def confirm_shape_proposals(
|
||||
project_id: int,
|
||||
snippet_id: int = 0,
|
||||
path: str = "",
|
||||
basis: str = "",
|
||||
min_score: float = 0.0,
|
||||
) -> dict:
|
||||
"""Confirm the proposer's canon proposals you have reviewed, in batch.
|
||||
|
||||
The machine proposes, judgment classifies (#2792): each matching row —
|
||||
live, unclassified, carrying a `proposal` with a snippet_id — becomes
|
||||
`instance` of that snippet, classified_by="agent", reason naming the
|
||||
basis and score. Narrow to what you actually looked at: at least one of
|
||||
snippet_id (confirm one canon's whole queue after reading its
|
||||
`list_shapes(proposal="canon", ...)` page), path (a directory you
|
||||
audited), or basis (e.g. "symbol" and "reference" are near-certain;
|
||||
"semantic" deserves a look first) is required — a bare confirm-all is
|
||||
not a judgment. min_score trims a basis's tail.
|
||||
|
||||
Proposals you do NOT confirm are judged with classify_shapes (variant,
|
||||
exempt, or instance of a different snippet) — any judgment retires the
|
||||
proposal. Requires write access. Returns {"confirmed": N}.
|
||||
"""
|
||||
uid = current_user_id()
|
||||
try:
|
||||
return await shape_ledger_svc.confirm_proposals(
|
||||
uid, project_id, snippet_id=snippet_id, path=path, basis=basis,
|
||||
min_score=min_score,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return {"error": str(exc)}
|
||||
|
||||
|
||||
async def refresh_pattern_coverage(project_id: int) -> dict:
|
||||
"""Seed or refresh the project's shape ledger NOW, and return the readout.
|
||||
|
||||
@@ -114,9 +161,17 @@ async def refresh_pattern_coverage(project_id: int) -> dict:
|
||||
owner adds one (Settings → Integrations → Git Forges); no served repo →
|
||||
bind_repo on a host a connection serves.
|
||||
|
||||
The refresh is also when the mechanical proposer runs (#2792): with the
|
||||
repo bodies in hand it matches every changed unclassified shape against
|
||||
canon and records proposals (see list_shapes proposal=), then regroups
|
||||
the derive-first candidates. Semantic matching is capped per refresh, so
|
||||
a large ledger's queue grows across refreshes rather than in one.
|
||||
|
||||
Returns the accounting payload — total, accounted, counts by status,
|
||||
unclassified, repos, largest_gaps — plus `pattern_coverage`, the same
|
||||
one-line summary enter_project carries.
|
||||
unclassified, repos, largest_gaps, `proposed` (canon proposals awaiting
|
||||
confirmation), `derive_groups` (the biggest repeats-with-no-canon
|
||||
families), `proposer` (what this refresh examined) — plus
|
||||
`pattern_coverage`, the same one-line summary enter_project carries.
|
||||
"""
|
||||
uid = current_user_id()
|
||||
coverage = await coverage_svc.refresh_for_caller(uid, project_id)
|
||||
@@ -127,5 +182,8 @@ async def refresh_pattern_coverage(project_id: int) -> dict:
|
||||
|
||||
|
||||
def register(mcp) -> None:
|
||||
for fn in (classify_shapes, list_shapes, refresh_pattern_coverage):
|
||||
for fn in (
|
||||
classify_shapes, list_shapes, refresh_pattern_coverage,
|
||||
confirm_shape_proposals,
|
||||
):
|
||||
mcp.tool(name=fn.__name__)(fn)
|
||||
|
||||
@@ -3,6 +3,7 @@ from datetime import datetime
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
@@ -19,6 +20,12 @@ from scribe.models.base import TimestampMixin
|
||||
SHAPE_STATUSES = ("canonical", "instance", "variant", "exempt", "unclassified")
|
||||
SHAPE_CLASSIFIERS = ("agent", "audit", "hook", "mechanical", "import")
|
||||
|
||||
# How the mechanical proposer (#2792) arrived at a proposal, strongest first.
|
||||
# `derive` is the odd one out: not "this is an instance of #N" but "this
|
||||
# shape repeats with NO canon — derive one first" (note 2786's derive-first
|
||||
# rule), so it carries a group key instead of a snippet.
|
||||
PROPOSAL_BASES = ("symbol", "text", "reference", "signature", "semantic", "derive")
|
||||
|
||||
|
||||
class CodeShape(Base, TimestampMixin):
|
||||
"""One extracted code shape and its classification against canon (#2787).
|
||||
@@ -42,6 +49,20 @@ class CodeShape(Base, TimestampMixin):
|
||||
snippet_id is SET NULL on snippet deletion: the classification's target
|
||||
is gone but the judgment happened; the sync pass (step 2) re-files such
|
||||
rows as unclassified so they rejoin the todo instead of dangling.
|
||||
|
||||
`signature` / `body_sha` (#2792) are the shape's content fingerprint —
|
||||
its definition line and a whitespace/comment-insensitive hash of its
|
||||
block — refreshed by every sync. They are what the mechanical proposer
|
||||
matches on and what a later drift recheck compares against; the ledger
|
||||
still never stores code bodies.
|
||||
|
||||
The proposal columns hold the proposer's standing suggestion for an
|
||||
UNCLASSIFIED row: `proposed_snippet_id` + `proposal_basis` + score for
|
||||
"looks like an instance of #N", or `proposal_basis="derive"` +
|
||||
`proposal_group` for "repeats with no canon". `proposed_sha` is the
|
||||
body_sha the row was last examined at, so a refresh re-examines only
|
||||
what changed. A judgment clears the proposal — the machine proposes,
|
||||
judgment classifies.
|
||||
"""
|
||||
|
||||
__tablename__ = "code_shapes"
|
||||
@@ -52,6 +73,7 @@ class CodeShape(Base, TimestampMixin):
|
||||
),
|
||||
Index("ix_code_shapes_project_status", "project_id", "status"),
|
||||
Index("ix_code_shapes_snippet", "snippet_id"),
|
||||
Index("ix_code_shapes_proposed", "project_id", "proposed_snippet_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
@@ -76,6 +98,31 @@ class CodeShape(Base, TimestampMixin):
|
||||
vanished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
signature: Mapped[str] = mapped_column(Text, default="")
|
||||
body_sha: Mapped[str] = mapped_column(Text, default="")
|
||||
proposed_snippet_id: Mapped[int | None] = mapped_column(
|
||||
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
proposal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
proposal_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
proposal_group: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
proposed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
proposed_sha: Mapped[str] = mapped_column(Text, default="")
|
||||
|
||||
@property
|
||||
def proposal(self) -> dict | None:
|
||||
"""The standing proposal as one object, or None when the proposer
|
||||
has nothing to say about this row."""
|
||||
if self.proposed_snippet_id is None and not self.proposal_group:
|
||||
return None
|
||||
out: dict = {"basis": self.proposal_basis, "score": self.proposal_score}
|
||||
if self.proposed_snippet_id is not None:
|
||||
out["snippet_id"] = self.proposed_snippet_id
|
||||
if self.proposal_group:
|
||||
out["group"] = self.proposal_group
|
||||
return out
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
@@ -93,6 +140,9 @@ class CodeShape(Base, TimestampMixin):
|
||||
"first_seen_commit": self.first_seen_commit,
|
||||
"last_seen_commit": self.last_seen_commit,
|
||||
"vanished_at": self.vanished_at.isoformat() if self.vanished_at else None,
|
||||
"signature": self.signature,
|
||||
"body_sha": self.body_sha,
|
||||
"proposal": self.proposal,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
@@ -1163,6 +1163,11 @@ async def _restore_v2(data: dict) -> dict:
|
||||
first_seen_commit=cs_data.get("first_seen_commit", ""),
|
||||
last_seen_commit=cs_data.get("last_seen_commit", ""),
|
||||
vanished_at=_dt(cs_data["vanished_at"]) if cs_data.get("vanished_at") else None,
|
||||
# Fingerprints restore; proposals (#2792) deliberately do not —
|
||||
# they are mechanical, and the next refresh recomputes them
|
||||
# against the restored snippet ids.
|
||||
signature=cs_data.get("signature", ""),
|
||||
body_sha=cs_data.get("body_sha", ""),
|
||||
created_at=_dt(cs_data.get("created_at")),
|
||||
updated_at=_dt(cs_data.get("updated_at")),
|
||||
))
|
||||
|
||||
+164
-28
@@ -25,12 +25,14 @@ only ever reads the cache.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import posixpath
|
||||
import re
|
||||
import tarfile
|
||||
from typing import NamedTuple
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from scribe.services.forge import ForgeSelector, get_forges
|
||||
@@ -91,6 +93,104 @@ _ARROW_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
class Definition(NamedTuple):
|
||||
"""One extracted definition with its content fingerprint (#2792).
|
||||
|
||||
`signature` is the definition line itself; `body_sha` hashes the block
|
||||
whitespace- and comment-insensitively; `body` is the block's text, held
|
||||
only for the duration of a refresh (the proposer matches on it) and
|
||||
never stored.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
name: str
|
||||
signature: str
|
||||
body_sha: str
|
||||
body: str
|
||||
|
||||
|
||||
def _definition_on(raw: str) -> tuple[str, str] | None:
|
||||
"""The (kind, name) this one line defines, or None. First match wins —
|
||||
the same order the hook's awk program tries."""
|
||||
m = _CSS_RE.match(raw)
|
||||
if m:
|
||||
return ("css", m.group(1))
|
||||
line = _MODIFIERS_RE.sub("", raw.lstrip())
|
||||
if m := _GO_METHOD_RE.match(line):
|
||||
return ("sym", m.group(1))
|
||||
if m := _KEYWORD_RE.match(line):
|
||||
name = m.group(1)
|
||||
if name.startswith("__") and name.endswith("__"):
|
||||
return None
|
||||
return ("sym", name)
|
||||
if m := _ARROW_RE.match(line):
|
||||
return ("sym", m.group(1))
|
||||
return None
|
||||
|
||||
|
||||
# A definition's block runs from its line until the next non-blank line at
|
||||
# its own indentation or shallower that is not a closer — so a Python def ends
|
||||
# at the next top-level statement, a braces block keeps its `}`, a CSS rule
|
||||
# keeps its `}`. Capped so a generated monolith can't make one shape's
|
||||
# fingerprint cover the file.
|
||||
_BLOCK_CAP = 120
|
||||
_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);")
|
||||
# Lines that don't change what a shape IS: comments and decorators. Dropped
|
||||
# from the fingerprint so touching a comment above the next function doesn't
|
||||
# read as this one's body changing.
|
||||
_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "<!--", "-->")
|
||||
_SIGNATURE_CAP = 300
|
||||
|
||||
|
||||
def _indent(line: str) -> int:
|
||||
return len(line) - len(line.lstrip())
|
||||
|
||||
|
||||
def _block_sha(lines: list[str]) -> str:
|
||||
kept = [
|
||||
" ".join(ln.split())
|
||||
for ln in lines
|
||||
if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES)
|
||||
]
|
||||
return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def extract_definitions(text: str) -> list[Definition]:
|
||||
"""Every definition this text makes, with signature + fingerprint.
|
||||
|
||||
Duplicate (kind, name) within one text collapse to the first — the
|
||||
ledger's identity is per file, so a second definition of the same name
|
||||
(an overload, a re-declaration) is the same shape to it.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
starts: list[tuple[int, str, str]] = []
|
||||
for i, raw in enumerate(lines):
|
||||
hit = _definition_on(raw)
|
||||
if hit:
|
||||
starts.append((i, hit[0], hit[1]))
|
||||
seen: set[tuple[str, str]] = set()
|
||||
out: list[Definition] = []
|
||||
for i, kind, name in starts:
|
||||
if (kind, name) in seen:
|
||||
continue
|
||||
seen.add((kind, name))
|
||||
base = _indent(lines[i])
|
||||
end = min(len(lines), i + _BLOCK_CAP)
|
||||
for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)):
|
||||
ln = lines[j]
|
||||
if not ln.strip():
|
||||
continue
|
||||
if _indent(ln) <= base and ln.strip() not in _CLOSERS:
|
||||
end = j
|
||||
break
|
||||
block = lines[i:end]
|
||||
out.append(Definition(
|
||||
kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(block),
|
||||
"\n".join(block),
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def extract_shapes(text: str) -> list[tuple[str, str]]:
|
||||
"""Every (kind, name) this text DEFINES — kind is "css" or "sym".
|
||||
|
||||
@@ -98,29 +198,7 @@ def extract_shapes(text: str) -> list[tuple[str, str]]:
|
||||
line, dunders are skipped (every class defines __init__ — guaranteed
|
||||
noise), duplicates within one text count once.
|
||||
"""
|
||||
seen: set[tuple[str, str]] = set()
|
||||
out: list[tuple[str, str]] = []
|
||||
for raw in text.splitlines():
|
||||
m = _CSS_RE.match(raw)
|
||||
if m:
|
||||
shape = ("css", m.group(1))
|
||||
else:
|
||||
line = _MODIFIERS_RE.sub("", raw.lstrip())
|
||||
if m := _GO_METHOD_RE.match(line):
|
||||
shape = ("sym", m.group(1))
|
||||
elif m := _KEYWORD_RE.match(line):
|
||||
name = m.group(1)
|
||||
if name.startswith("__") and name.endswith("__"):
|
||||
continue
|
||||
shape = ("sym", name)
|
||||
elif m := _ARROW_RE.match(line):
|
||||
shape = ("sym", m.group(1))
|
||||
else:
|
||||
continue
|
||||
if shape not in seen:
|
||||
seen.add(shape)
|
||||
out.append(shape)
|
||||
return out
|
||||
return [(d.kind, d.name) for d in extract_definitions(text)]
|
||||
|
||||
|
||||
def scannable(path: str) -> bool:
|
||||
@@ -131,14 +209,33 @@ def scannable(path: str) -> bool:
|
||||
return not path.lower().endswith(_SKIP_SUFFIXES)
|
||||
|
||||
|
||||
class ArchiveShape(NamedTuple):
|
||||
"""A definition located in a repo archive — what the sync upserts and
|
||||
the proposer matches. The leading (path, kind, name) triple is the
|
||||
ledger identity; the rest is the fingerprint and the transient body."""
|
||||
|
||||
path: str
|
||||
kind: str
|
||||
name: str
|
||||
signature: str
|
||||
body_sha: str
|
||||
body: str
|
||||
|
||||
|
||||
def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
|
||||
"""(path, kind, name) for every definition in a repo tarball.
|
||||
"""(path, kind, name) for every definition in a repo tarball — the
|
||||
identity view of definitions_from_archive."""
|
||||
return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)]
|
||||
|
||||
|
||||
def definitions_from_archive(blob: bytes) -> list[ArchiveShape]:
|
||||
"""Every definition in a repo tarball, with its fingerprint and body.
|
||||
|
||||
Forge archives wrap content in a single top-level directory (repo-ref/);
|
||||
that component is stripped so paths match recorded snippet locations,
|
||||
which are repo-relative. Non-UTF-8 files are binaries and skipped.
|
||||
"""
|
||||
shapes: list[tuple[str, str, str]] = []
|
||||
shapes: list[ArchiveShape] = []
|
||||
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
|
||||
for member in tar:
|
||||
if not member.isfile() or "/" not in member.name:
|
||||
@@ -153,7 +250,10 @@ def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
|
||||
text = handle.read().decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
shapes.extend((path, kind, name) for kind, name in extract_shapes(text))
|
||||
shapes.extend(
|
||||
ArchiveShape(path, d.kind, d.name, d.signature, d.body_sha, d.body)
|
||||
for d in extract_definitions(text)
|
||||
)
|
||||
return shapes
|
||||
|
||||
|
||||
@@ -253,13 +353,17 @@ async def compute_coverage(
|
||||
|
||||
served: list[tuple[str, str]] = []
|
||||
recorded = await _recorded_locations(user_id, project_id)
|
||||
# The proposer's canon catalog, read once per refresh and shared across
|
||||
# the project's repos (#2792).
|
||||
canons = None
|
||||
proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0}
|
||||
for key in await keys_for_project(user_id, project_id):
|
||||
hit = selector.resolve(key)
|
||||
if hit is None:
|
||||
continue # bound to a host no connection serves
|
||||
forge, api_repo = hit
|
||||
ref = await forge.default_branch(api_repo)
|
||||
shapes = shapes_from_archive(await forge.archive(api_repo, ref))
|
||||
definitions = definitions_from_archive(await forge.archive(api_repo, ref))
|
||||
# The head commit is provenance sugar on the ledger rows; failing to
|
||||
# learn it must not fail the sync — the ref names the point well
|
||||
# enough and the row timestamps carry the when.
|
||||
@@ -268,13 +372,31 @@ async def compute_coverage(
|
||||
except ForgeError:
|
||||
marker = ref
|
||||
await shape_ledger.sync_repo_shapes(
|
||||
project_id, key, shapes, seen_marker=marker
|
||||
project_id, key, definitions, seen_marker=marker
|
||||
)
|
||||
served.append((key, ref))
|
||||
# Propose while the bodies are in hand — the one moment they exist.
|
||||
# Canonical marking below only touches rows the proposer leaves
|
||||
# alone (a canon's own location never gets a proposal), so the order
|
||||
# is immaterial; the proposer must not be able to fail the refresh.
|
||||
try:
|
||||
if canons is None:
|
||||
canons = await shape_ledger.canon_catalog(user_id)
|
||||
stats = await shape_ledger.propose_for_repo(
|
||||
user_id, project_id, key, definitions, canons=canons
|
||||
)
|
||||
for k in proposer_stats:
|
||||
proposer_stats[k] += stats.get(k, 0)
|
||||
except Exception:
|
||||
logger.warning("shape proposer failed for %s", key, exc_info=True)
|
||||
if not served:
|
||||
return None
|
||||
|
||||
await shape_ledger.mark_canonicals(project_id, recorded)
|
||||
try:
|
||||
await shape_ledger.apply_derive_groups(project_id)
|
||||
except Exception:
|
||||
logger.warning("derive-first grouping failed", exc_info=True)
|
||||
|
||||
# Project-wide readout, deliberately wider than this walk: a second bound
|
||||
# repo that was unreachable today still has live rows, and they count.
|
||||
@@ -290,11 +412,17 @@ async def compute_coverage(
|
||||
agg["accounted"] += row.status != "unclassified"
|
||||
|
||||
unclassified = counts.pop("unclassified")
|
||||
proposals = shape_ledger.proposal_summary(rows)
|
||||
return {
|
||||
"total": len(rows),
|
||||
"accounted": len(rows) - unclassified,
|
||||
"unclassified": unclassified,
|
||||
"counts": counts,
|
||||
# The proposer's standing (#2792): canon proposals awaiting a
|
||||
# confirm, the largest derive-first groups, and what this refresh did.
|
||||
"proposed": proposals["proposed"],
|
||||
"derive_groups": proposals["derive_groups"],
|
||||
"proposer": proposer_stats,
|
||||
# Honesty flag, not decoration: every surface that shows the number
|
||||
# is expected to carry it through.
|
||||
"estimate": True,
|
||||
@@ -438,6 +566,14 @@ def coverage_line(coverage: dict) -> str:
|
||||
unclassified = coverage.get("unclassified", 0)
|
||||
if unclassified:
|
||||
line += f"; {unclassified} unclassified"
|
||||
standing = []
|
||||
if coverage.get("proposed"):
|
||||
standing.append(f"{coverage['proposed']} proposed")
|
||||
n_groups = len(coverage.get("derive_groups") or [])
|
||||
if n_groups:
|
||||
standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}")
|
||||
if standing:
|
||||
line += f" ({', '.join(standing)})"
|
||||
gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
|
||||
if gaps:
|
||||
line += ", largest: " + ", ".join(gaps)
|
||||
|
||||
@@ -21,9 +21,11 @@ an extracted shape.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Iterable, NamedTuple
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
@@ -65,15 +67,17 @@ def location_covers(loc_path: str, loc_symbol: str, path: str, name: str) -> boo
|
||||
async def sync_repo_shapes(
|
||||
project_id: int,
|
||||
repo_key: str,
|
||||
shapes: list[tuple[str, str, str]],
|
||||
shapes: list,
|
||||
*,
|
||||
seen_marker: str,
|
||||
) -> None:
|
||||
"""Upsert one repo's extracted (path, kind, name) shapes into the ledger.
|
||||
"""Upsert one repo's extracted shapes into the ledger.
|
||||
|
||||
``seen_marker`` is the commit the archive was read at when the forge can
|
||||
say, else the ref name — provenance sugar; the row timestamps carry the
|
||||
when.
|
||||
``shapes`` are (path, kind, name) triples, or the richer ArchiveShape
|
||||
records (#2792) whose 4th/5th fields — signature, body_sha — refresh the
|
||||
row's content fingerprint. ``seen_marker`` is the commit the archive was
|
||||
read at when the forge can say, else the ref name — provenance sugar;
|
||||
the row timestamps carry the when.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
@@ -87,7 +91,10 @@ async def sync_repo_shapes(
|
||||
).scalars().all()
|
||||
by_key = {(r.path, r.symbol, r.kind): r for r in rows}
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
for path, kind, name in shapes:
|
||||
for shape in shapes:
|
||||
path, kind, name = shape[0], shape[1], shape[2]
|
||||
signature = shape[3] if len(shape) > 3 else ""
|
||||
body_sha = shape[4] if len(shape) > 4 else ""
|
||||
key = (path, name, kind)
|
||||
if key in seen:
|
||||
continue
|
||||
@@ -98,9 +105,14 @@ async def sync_repo_shapes(
|
||||
project_id=project_id, repo_key=repo_key,
|
||||
path=path, symbol=name, kind=kind,
|
||||
first_seen_commit=seen_marker, last_seen_commit=seen_marker,
|
||||
signature=signature, body_sha=body_sha,
|
||||
))
|
||||
continue
|
||||
row.last_seen_commit = seen_marker
|
||||
if signature:
|
||||
row.signature = signature
|
||||
if body_sha:
|
||||
row.body_sha = body_sha
|
||||
# A shape that vanished and came back is live again — the vanish
|
||||
# stays visible in history via updated_at, not as a dead flag.
|
||||
row.vanished_at = None
|
||||
@@ -297,6 +309,10 @@ async def classify_shapes(
|
||||
status = item["status"]
|
||||
for row in matches:
|
||||
row.status = status
|
||||
# The machine proposes, judgment classifies: any judgment
|
||||
# retires the standing proposal; a withdrawal also forgets
|
||||
# the examination so the next refresh proposes afresh.
|
||||
_clear_proposal(row, reexamine=(status == "unclassified"))
|
||||
if status == "unclassified":
|
||||
row.snippet_id = None
|
||||
row.reason = None
|
||||
@@ -324,12 +340,16 @@ async def list_project_shapes(
|
||||
include_vanished: bool = False,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
proposal: str = "",
|
||||
) -> tuple[list[CodeShape], int]:
|
||||
"""A filtered page of a project's ledger, with the unfiltered-match total.
|
||||
|
||||
([], 0) when the caller can't read the project — the same silence every
|
||||
other project list gives. ``path`` matches the exact file or anything
|
||||
beneath it, mirroring recorded-location semantics.
|
||||
beneath it, mirroring recorded-location semantics. ``proposal`` narrows
|
||||
to rows the proposer has spoken about: "any", "canon" (an instance-of-#N
|
||||
suggestion), "derive" (a repeats-with-no-canon group), or one basis
|
||||
name (symbol/reference/text/signature/semantic).
|
||||
"""
|
||||
from sqlalchemy import func, or_
|
||||
|
||||
@@ -349,6 +369,17 @@ async def list_project_shapes(
|
||||
))
|
||||
if snippet_id:
|
||||
conds.append(CodeShape.snippet_id == snippet_id)
|
||||
if proposal == "any":
|
||||
conds.append(or_(
|
||||
CodeShape.proposed_snippet_id.isnot(None),
|
||||
CodeShape.proposal_group.isnot(None),
|
||||
))
|
||||
elif proposal == "canon":
|
||||
conds.append(CodeShape.proposed_snippet_id.isnot(None))
|
||||
elif proposal == "derive":
|
||||
conds.append(CodeShape.proposal_group.isnot(None))
|
||||
elif proposal:
|
||||
conds.append(CodeShape.proposal_basis == proposal)
|
||||
async with async_session() as session:
|
||||
total = (
|
||||
await session.execute(
|
||||
@@ -589,6 +620,7 @@ async def stamp_write_path_instances(
|
||||
row.reason = why
|
||||
row.classified_by = "hook"
|
||||
row.classified_at = now
|
||||
_clear_proposal(row)
|
||||
stamped.append({
|
||||
"path": path, "symbol": name, "kind": kind,
|
||||
"snippet_id": sid, "reason": why,
|
||||
@@ -596,3 +628,431 @@ async def stamp_write_path_instances(
|
||||
if stamped:
|
||||
await session.commit()
|
||||
return stamped
|
||||
|
||||
|
||||
# --- the mechanical proposer (#2792): the machine proposes, judgment classifies
|
||||
#
|
||||
# Runs inside the coverage refresh — the one moment the shape BODIES exist
|
||||
# (the archive is in memory; the ledger stores fingerprints, never code).
|
||||
# Over every live unclassified row whose content changed since it was last
|
||||
# examined, it tries, strongest first:
|
||||
#
|
||||
# symbol the shape bears a recorded canon's symbol at another location
|
||||
# (a second definition of the canon's name — an instance or a
|
||||
# duplicate to consolidate; either way, it answers to #N);
|
||||
# text whitespace-insensitive containment either way between the
|
||||
# body and the canon's recorded code (git-grep, in effect —
|
||||
# a verbatim copy, or the canon's own call-site example);
|
||||
# reference the body calls/uses a canon's symbol — the call-site shape
|
||||
# the P7 backfill classified by hand (#2790);
|
||||
# signature a sym whose definition line, name blanked, resembles the
|
||||
# canon's (the move_*/resequence family shape);
|
||||
# semantic the widest net: the body (concept-queried, as the write-path
|
||||
# arm does) scores above the write-path threshold against a
|
||||
# canon — capped per refresh, so the cost is bounded and a big
|
||||
# ledger is worked through across refreshes.
|
||||
#
|
||||
# A hit is a PROPOSAL on the row (proposed_snippet_id/basis/score), never a
|
||||
# classification: an agent confirms in batches (confirm_proposals) or judges
|
||||
# otherwise (classify_shapes clears it). Rows with no canon hit are grouped
|
||||
# by the derive-first rule (note 2786): the same body fingerprint in ≥2
|
||||
# places, or the same name defined in ≥3 files, is "a repeating shape with
|
||||
# no canon — derive one first", carried as proposal_basis="derive" + a group
|
||||
# key so sessions see the consolidation candidates as one thing.
|
||||
|
||||
# Derive-first floors. Identical bodies twice is already a copy; a bare name
|
||||
# needs more repetition before it reads as a family (setup/handler/register
|
||||
# recur by convention, not by duplication).
|
||||
_DERIVE_MIN_DUP = 2
|
||||
_DERIVE_MIN_NAME = 3
|
||||
# Semantic checks per repo per refresh — an embedding each (local fastembed),
|
||||
# bounded so a 4,000-row ledger is worked through over refreshes, not in one.
|
||||
_SEMANTIC_CAP = 150
|
||||
# Signature resemblance floor, name blanked (difflib ratio) — and a length
|
||||
# floor, because `def NAME():` resembles `def NAME(x):` at 0.95 while saying
|
||||
# nothing; a family shape has parameters to resemble.
|
||||
_SIGNATURE_FLOOR = 0.8
|
||||
_SIGNATURE_MIN_LEN = 30
|
||||
# Textual containment needs enough substance to mean anything.
|
||||
_TEXT_FLOOR = 40
|
||||
_BASIS_ORDER = ("symbol", "text", "reference", "signature")
|
||||
|
||||
|
||||
class Canon(NamedTuple):
|
||||
snippet_id: int
|
||||
kind: str
|
||||
symbol: str
|
||||
locations: tuple[tuple[str, str], ...]
|
||||
signature: str
|
||||
code_norm: str
|
||||
|
||||
|
||||
def _norm_text(text: str) -> str:
|
||||
return " ".join((text or "").split())
|
||||
|
||||
|
||||
def signature_similarity(sig_a: str, name_a: str, sig_b: str, name_b: str) -> float:
|
||||
"""How alike two definition lines are once their own names are blanked —
|
||||
`def move_event(project_id, event_id, after_id)` against
|
||||
`def move_beat(project_id, beat_id, after_id)` reads high."""
|
||||
a = _norm_text(sig_a)
|
||||
b = _norm_text(sig_b)
|
||||
if name_a:
|
||||
a = a.replace(name_a, "NAME")
|
||||
if name_b:
|
||||
b = b.replace(name_b, "NAME")
|
||||
if len(a) < _SIGNATURE_MIN_LEN or len(b) < _SIGNATURE_MIN_LEN:
|
||||
return 0.0
|
||||
return difflib.SequenceMatcher(None, a, b).ratio()
|
||||
|
||||
|
||||
def text_contains(body: str, code: str) -> bool:
|
||||
"""Whitespace-insensitive containment, either way, above a substance
|
||||
floor — the proposer's git-grep."""
|
||||
a = _norm_text(body)
|
||||
b = _norm_text(code)
|
||||
if len(a) < _TEXT_FLOOR or len(b) < _TEXT_FLOOR:
|
||||
return False
|
||||
return a in b or b in a
|
||||
|
||||
|
||||
def match_canon(
|
||||
kind: str, path: str, symbol: str, signature: str, body: str,
|
||||
canons: Iterable[Canon],
|
||||
) -> tuple[int, str, float] | None:
|
||||
"""The strongest (snippet_id, basis, score) a shape earns against the
|
||||
canon catalog, by the non-semantic bases — or None. Strongest by
|
||||
basis order, then by score within the basis."""
|
||||
best: dict[str, tuple[float, int]] = {}
|
||||
|
||||
def offer(basis: str, score: float, sid: int) -> None:
|
||||
cur = best.get(basis)
|
||||
if cur is None or score > cur[0]:
|
||||
best[basis] = (score, sid)
|
||||
|
||||
norm_sym = _norm_symbol(symbol)
|
||||
for c in canons:
|
||||
if c.kind != kind:
|
||||
continue
|
||||
if c.symbol and _norm_symbol(c.symbol) == norm_sym:
|
||||
if not any(location_covers(lp, ls, path, symbol) for lp, ls in c.locations):
|
||||
offer("symbol", 1.0, c.snippet_id)
|
||||
continue # its own location is canonical territory, not a proposal
|
||||
if c.symbol and references_symbol(body, c.symbol, kind):
|
||||
offer("reference", 0.9, c.snippet_id)
|
||||
if c.code_norm and text_contains(body, c.code_norm):
|
||||
offer("text", 0.95, c.snippet_id)
|
||||
if kind == "sym" and c.signature:
|
||||
ratio = signature_similarity(signature, symbol, c.signature, c.symbol)
|
||||
if ratio >= _SIGNATURE_FLOOR:
|
||||
offer("signature", round(ratio, 3), c.snippet_id)
|
||||
for basis in _BASIS_ORDER:
|
||||
if basis in best:
|
||||
score, sid = best[basis]
|
||||
return (sid, basis, score)
|
||||
return None
|
||||
|
||||
|
||||
async def canon_catalog(user_id: int) -> list[Canon]:
|
||||
"""Every snippet this user can browse, as matchable canon — own projects
|
||||
and shared ones alike, because family canon counts (note 2786)."""
|
||||
from scribe.models.note import Note
|
||||
from scribe.services.access import browsable_notes_clause
|
||||
from scribe.services.coverage import extract_definitions
|
||||
from scribe.services.snippets import SNIPPET_NOTE_TYPE, snippet_fields
|
||||
|
||||
async with async_session() as session:
|
||||
notes = (
|
||||
await session.execute(
|
||||
select(Note).where(
|
||||
browsable_notes_clause(user_id),
|
||||
Note.note_type == SNIPPET_NOTE_TYPE,
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
out: list[Canon] = []
|
||||
for note in notes:
|
||||
fields = snippet_fields(note)
|
||||
symbol = (fields.get("symbol") or "").strip()
|
||||
kind = snippet_kind(symbol, fields.get("language") or "")
|
||||
code = fields.get("code") or ""
|
||||
defs = extract_definitions(code)
|
||||
signature = ""
|
||||
if defs:
|
||||
own = next(
|
||||
(d for d in defs if _norm_symbol(d.name) == _norm_symbol(symbol)), defs[0]
|
||||
)
|
||||
signature = own.signature
|
||||
out.append(Canon(
|
||||
int(note.id), kind, symbol,
|
||||
tuple(
|
||||
((loc.get("path") or ""), (loc.get("symbol") or ""))
|
||||
for loc in fields.get("locations") or []
|
||||
),
|
||||
signature, _norm_text(code),
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def _clear_proposal(row: CodeShape, *, reexamine: bool = False) -> None:
|
||||
row.proposed_snippet_id = None
|
||||
row.proposal_basis = None
|
||||
row.proposal_score = None
|
||||
row.proposal_group = None
|
||||
if reexamine:
|
||||
row.proposed_at = None
|
||||
row.proposed_sha = ""
|
||||
|
||||
|
||||
def _substance(text: str) -> int:
|
||||
return len("".join((text or "").split()))
|
||||
|
||||
|
||||
async def _semantic_canon(
|
||||
user_id: int, body: str, allowed: set[int]
|
||||
) -> tuple[int, float] | None:
|
||||
from scribe.services.embeddings import semantic_search_notes
|
||||
from scribe.services.plugin_context import (
|
||||
WRITEPATH_DEFAULT_THRESHOLD, WRITEPATH_MIN_CODE_CHARS, concept_query,
|
||||
)
|
||||
|
||||
if _substance(body) < WRITEPATH_MIN_CODE_CHARS or not allowed:
|
||||
return None
|
||||
query = concept_query(body) or body
|
||||
hits = await semantic_search_notes(
|
||||
user_id, query, limit=3, threshold=WRITEPATH_DEFAULT_THRESHOLD,
|
||||
note_type="snippet", scope="browse",
|
||||
)
|
||||
for score, note in hits:
|
||||
if int(note.id) in allowed:
|
||||
return int(note.id), round(float(score), 3)
|
||||
return None
|
||||
|
||||
|
||||
async def propose_for_repo(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
repo_key: str,
|
||||
definitions: list,
|
||||
*,
|
||||
canons: list[Canon] | None = None,
|
||||
semantic_cap: int = _SEMANTIC_CAP,
|
||||
) -> dict:
|
||||
"""Examine one repo's live unclassified rows against canon and record
|
||||
proposals. ``definitions`` are the ArchiveShape records the sync just
|
||||
upserted (their bodies are the matching material). Rows whose fingerprint
|
||||
is unchanged since their last examination are skipped; rows that only
|
||||
the capped semantic pass could not reach stay unexamined, so the next
|
||||
refresh reaches the next slice. Returns counts."""
|
||||
if canons is None:
|
||||
canons = await canon_catalog(user_id)
|
||||
by_key = {(d[0], d[1], d[2]): d for d in definitions}
|
||||
sym_canon_ids = {c.snippet_id for c in canons if c.kind == "sym"}
|
||||
now = datetime.now(timezone.utc)
|
||||
examined = proposed = checked = 0
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(CodeShape).where(
|
||||
CodeShape.project_id == project_id,
|
||||
CodeShape.repo_key == repo_key,
|
||||
CodeShape.status == "unclassified",
|
||||
CodeShape.vanished_at.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
semantic_todo: list[tuple[CodeShape, object]] = []
|
||||
for row in rows:
|
||||
d = by_key.get((row.path, row.kind, row.symbol))
|
||||
if d is None:
|
||||
continue
|
||||
signature, body_sha, body = d[3], d[4], d[5]
|
||||
if row.proposed_at is not None and row.proposed_sha == body_sha:
|
||||
continue
|
||||
examined += 1
|
||||
group = row.proposal_group # derive grouping is reassigned below
|
||||
hit = match_canon(row.kind, row.path, row.symbol, signature, body, canons)
|
||||
_clear_proposal(row)
|
||||
row.proposal_group = group
|
||||
row.proposed_at = now
|
||||
row.proposed_sha = body_sha
|
||||
if hit:
|
||||
row.proposed_snippet_id, row.proposal_basis, row.proposal_score = hit
|
||||
row.proposal_group = None
|
||||
proposed += 1
|
||||
elif row.kind == "sym":
|
||||
semantic_todo.append((row, d))
|
||||
for i, (row, d) in enumerate(semantic_todo):
|
||||
if i >= semantic_cap:
|
||||
# Not reached this refresh: leave it unexamined so the next
|
||||
# refresh picks it up, rather than stamping a false "nothing".
|
||||
row.proposed_at = None
|
||||
row.proposed_sha = ""
|
||||
continue
|
||||
checked += 1
|
||||
try:
|
||||
found = await _semantic_canon(user_id, d[5], sym_canon_ids)
|
||||
except Exception:
|
||||
logger.warning("semantic proposal failed", exc_info=True)
|
||||
found = None
|
||||
if found:
|
||||
row.proposed_snippet_id, row.proposal_score = found
|
||||
row.proposal_basis = "semantic"
|
||||
row.proposal_group = None
|
||||
proposed += 1
|
||||
await session.commit()
|
||||
return {"examined": examined, "proposed": proposed, "semantic_checked": checked}
|
||||
|
||||
|
||||
def derive_groups(
|
||||
rows: Iterable[tuple[str, str, str, str]]
|
||||
) -> dict[tuple[str, str, str], str]:
|
||||
"""The derive-first grouping over (path, kind, symbol, body_sha) rows
|
||||
that matched no canon: {(path, kind, symbol): group_key}. Identical
|
||||
bodies in ≥2 places group as `dup:<sha>`; the same name defined in ≥3
|
||||
files groups as `name:<kind>:<symbol>`; a row joins at most one group,
|
||||
the copy before the name."""
|
||||
by_sha: dict[str, list[tuple[str, str, str]]] = {}
|
||||
by_name: dict[tuple[str, str], list[tuple[str, str, str]]] = {}
|
||||
for path, kind, symbol, sha in rows:
|
||||
key = (path, kind, symbol)
|
||||
if sha:
|
||||
by_sha.setdefault(sha, []).append(key)
|
||||
by_name.setdefault((kind, _norm_symbol(symbol)), []).append(key)
|
||||
out: dict[tuple[str, str, str], str] = {}
|
||||
for sha, keys in by_sha.items():
|
||||
if len(set(keys)) >= _DERIVE_MIN_DUP:
|
||||
for key in keys:
|
||||
out.setdefault(key, f"dup:{sha}")
|
||||
for (kind, symbol), keys in by_name.items():
|
||||
if len({k[0] for k in keys}) >= _DERIVE_MIN_NAME:
|
||||
for key in keys:
|
||||
out.setdefault(key, f"name:{kind}:{symbol}")
|
||||
return out
|
||||
|
||||
|
||||
async def apply_derive_groups(project_id: int) -> int:
|
||||
"""Recompute derive-first groups over the project's live unclassified
|
||||
rows that carry no canon proposal; returns how many rows are grouped."""
|
||||
now = datetime.now(timezone.utc)
|
||||
grouped = 0
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(CodeShape).where(
|
||||
CodeShape.project_id == project_id,
|
||||
CodeShape.status == "unclassified",
|
||||
CodeShape.vanished_at.is_(None),
|
||||
CodeShape.proposed_snippet_id.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
groups = derive_groups(
|
||||
(r.path, r.kind, r.symbol, r.body_sha or "") for r in rows
|
||||
)
|
||||
sizes: dict[str, int] = {}
|
||||
for g in groups.values():
|
||||
sizes[g] = sizes.get(g, 0) + 1
|
||||
for row in rows:
|
||||
key = groups.get((row.path, row.kind, row.symbol))
|
||||
if key:
|
||||
row.proposal_basis = "derive"
|
||||
row.proposal_group = key
|
||||
row.proposal_score = float(sizes[key])
|
||||
if row.proposed_at is None:
|
||||
row.proposed_at = now
|
||||
grouped += 1
|
||||
elif row.proposal_group:
|
||||
row.proposal_basis = None
|
||||
row.proposal_group = None
|
||||
row.proposal_score = None
|
||||
await session.commit()
|
||||
return grouped
|
||||
|
||||
|
||||
def proposal_summary(rows: Iterable[CodeShape], *, top: int = 8) -> dict:
|
||||
"""The readout's view of the proposer's standing: how many canon
|
||||
proposals await confirmation, and the largest derive-first groups."""
|
||||
proposed = 0
|
||||
groups: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
if row.status != "unclassified":
|
||||
continue
|
||||
if row.proposed_snippet_id is not None:
|
||||
proposed += 1
|
||||
elif row.proposal_group:
|
||||
g = groups.setdefault(row.proposal_group, {
|
||||
"group": row.proposal_group, "kind": row.kind,
|
||||
"label": (
|
||||
("." if row.kind == "css" else "") + row.symbol
|
||||
if row.proposal_group.startswith("name:")
|
||||
else f"{row.symbol} (identical body)"
|
||||
),
|
||||
"size": 0, "paths": [],
|
||||
})
|
||||
g["size"] += 1
|
||||
if len(g["paths"]) < 3:
|
||||
g["paths"].append(row.path)
|
||||
ranked = sorted(groups.values(), key=lambda g: (-g["size"], g["group"]))
|
||||
return {"proposed": proposed, "derive_groups": ranked[:top]}
|
||||
|
||||
|
||||
async def confirm_proposals(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
*,
|
||||
snippet_id: int = 0,
|
||||
path: str = "",
|
||||
basis: str = "",
|
||||
min_score: float = 0.0,
|
||||
) -> dict:
|
||||
"""Turn reviewed canon proposals into `instance` rows, in one batch.
|
||||
|
||||
At least one of snippet_id / path / basis must narrow the batch — "confirm
|
||||
everything proposed" without having looked is not a judgment. Each row
|
||||
becomes instance-of-its-proposed-snippet, classified_by="agent", reason
|
||||
naming the basis and score; the proposal is retired. Returns
|
||||
{"confirmed": N}."""
|
||||
from scribe.services import access
|
||||
|
||||
if not (snippet_id or path.strip() or basis.strip()):
|
||||
raise ValueError(
|
||||
"name what you reviewed: confirm by snippet_id, path, and/or basis"
|
||||
)
|
||||
if not await access.can_write_project(user_id, project_id):
|
||||
raise ValueError(f"project {project_id} not found or no write access")
|
||||
from sqlalchemy import or_
|
||||
|
||||
conds = [
|
||||
CodeShape.project_id == project_id,
|
||||
CodeShape.status == "unclassified",
|
||||
CodeShape.vanished_at.is_(None),
|
||||
CodeShape.proposed_snippet_id.isnot(None),
|
||||
]
|
||||
if snippet_id:
|
||||
conds.append(CodeShape.proposed_snippet_id == snippet_id)
|
||||
if path.strip():
|
||||
clean = path.strip().strip("/")
|
||||
conds.append(or_(CodeShape.path == clean, CodeShape.path.like(clean + "/%")))
|
||||
if basis.strip():
|
||||
conds.append(CodeShape.proposal_basis == basis.strip())
|
||||
now = datetime.now(timezone.utc)
|
||||
confirmed = 0
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(select(CodeShape).where(*conds))).scalars().all()
|
||||
for row in rows:
|
||||
if (row.proposal_score or 0.0) < min_score:
|
||||
continue
|
||||
row.status = "instance"
|
||||
row.snippet_id = row.proposed_snippet_id
|
||||
row.reason = (
|
||||
f"confirmed {row.proposal_basis} proposal"
|
||||
f" ({(row.proposal_score or 0.0):.2f})"
|
||||
)
|
||||
row.classified_by = "agent"
|
||||
row.classified_at = now
|
||||
_clear_proposal(row)
|
||||
confirmed += 1
|
||||
await session.commit()
|
||||
return {"confirmed": confirmed}
|
||||
|
||||
@@ -351,3 +351,145 @@ async def test_recent_pulls_reads_the_usage_stream(seeded):
|
||||
assert sid in pulls
|
||||
assert sid + 1000 not in pulls
|
||||
assert sid + 2000 not in pulls
|
||||
|
||||
|
||||
# --- #2792: the mechanical proposer against real rows -------------------------
|
||||
|
||||
|
||||
def _quiet_semantic():
|
||||
"""The semantic basis needs the embedder; these tests prove the other
|
||||
bases and the bookkeeping, so it answers "nothing" here."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from scribe.services import shape_ledger
|
||||
return patch.object(shape_ledger, "_semantic_canon", AsyncMock(return_value=None))
|
||||
|
||||
|
||||
def _defs(*items):
|
||||
"""ArchiveShape-like records: (path, kind, name, signature, body_sha, body)."""
|
||||
import hashlib
|
||||
out = []
|
||||
for path, kind, name, signature, body in items:
|
||||
sha = hashlib.sha1(" ".join(body.split()).encode()).hexdigest()[:16]
|
||||
out.append((path, kind, name, signature, sha, body))
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_proposer_proposes_and_confirm_classifies(seeded):
|
||||
"""Bodies in hand, the proposer records proposals on unclassified rows —
|
||||
symbol (a second `factory` elsewhere), reference (a call site), and
|
||||
nothing for the unrelated — skips rows whose content it already judged,
|
||||
and a scoped confirm turns proposals into agent instances while a
|
||||
classify on another retires its proposal."""
|
||||
from scribe.services.shape_ledger import (
|
||||
confirm_proposals, propose_for_repo,
|
||||
)
|
||||
|
||||
owner, other, pid, sid = (
|
||||
seeded["owner"], seeded["other"], seeded["pid"], seeded["snippet"]
|
||||
)
|
||||
defs = _defs(
|
||||
("src/app.py", "sym", "make_app", "def make_app():", "def make_app():\n app = factory()\n return app"),
|
||||
("src/app.py", "sym", "Config", "class Config:", "class Config:\n debug = False"),
|
||||
("src/util.py", "sym", "helper", "def helper(x):", "def helper(x):\n return x"),
|
||||
("src/dup.py", "sym", "factory", "def factory():", "def factory():\n return 1"),
|
||||
("web/button.css", "css", "btn", ".btn {", ".btn {\n color: red;\n}"),
|
||||
)
|
||||
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
|
||||
|
||||
with _quiet_semantic():
|
||||
stats = await propose_for_repo(owner, pid, REPO, defs)
|
||||
assert stats == {"examined": 5, "proposed": 2, "semantic_checked": 2}
|
||||
rows, total = await list_project_shapes(owner, pid, proposal="canon")
|
||||
by_symbol = {r.symbol: r for r in rows}
|
||||
assert total == 2
|
||||
assert by_symbol["factory"].proposal == {"basis": "symbol", "score": 1.0, "snippet_id": sid}
|
||||
assert by_symbol["make_app"].proposal == {"basis": "reference", "score": 0.9, "snippet_id": sid}
|
||||
rows, _ = await list_project_shapes(owner, pid, proposal="reference")
|
||||
assert [r.symbol for r in rows] == ["make_app"]
|
||||
|
||||
# Same content again → nothing re-examined (the semantic cap would
|
||||
# otherwise be spent on the same rows every refresh). A cap that leaves
|
||||
# rows unreached leaves them UNexamined, so the next refresh gets them.
|
||||
with _quiet_semantic():
|
||||
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 0
|
||||
await classify_shapes(owner, pid, [
|
||||
{"path": "src/util.py", "symbol": "helper", "status": "unclassified"},
|
||||
])
|
||||
assert (await propose_for_repo(owner, pid, REPO, defs, semantic_cap=0))["semantic_checked"] == 0
|
||||
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 1
|
||||
|
||||
# Outsider can't confirm; the owner confirms by snippet, scoped.
|
||||
with pytest.raises(ValueError):
|
||||
await confirm_proposals(other, pid, snippet_id=sid)
|
||||
assert await confirm_proposals(owner, pid, basis="symbol") == {"confirmed": 1}
|
||||
rows, _ = await list_project_shapes(owner, pid, snippet_id=sid)
|
||||
factory = next(r for r in rows if r.symbol == "factory")
|
||||
assert factory.status == "instance" and factory.classified_by == "agent"
|
||||
assert factory.reason == "confirmed symbol proposal (1.00)"
|
||||
assert factory.proposal is None
|
||||
|
||||
# A judgment on a proposed row retires the proposal; withdrawing a
|
||||
# judgment forgets the examination so the next pass proposes afresh.
|
||||
await classify_shapes(owner, pid, [
|
||||
{"path": "src/app.py", "symbol": "make_app", "status": "exempt", "reason": "bootstrap"},
|
||||
])
|
||||
rows, _ = await list_project_shapes(owner, pid, path="src/app.py")
|
||||
make_app = next(r for r in rows if r.symbol == "make_app")
|
||||
assert make_app.status == "exempt" and make_app.proposal is None
|
||||
await classify_shapes(owner, pid, [
|
||||
{"path": "src/app.py", "symbol": "make_app", "status": "unclassified"},
|
||||
])
|
||||
with _quiet_semantic():
|
||||
assert (await propose_for_repo(owner, pid, REPO, defs))["proposed"] == 1
|
||||
rows, _ = await list_project_shapes(owner, pid, proposal="canon")
|
||||
assert [r.symbol for r in rows] == ["make_app"]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_derive_groups_land_on_rows_and_in_the_summary(seeded):
|
||||
"""Shapes with no canon hit that repeat — identical bodies in two files,
|
||||
the same name in three — carry a derive proposal, and the readout ranks
|
||||
the families. A canon proposal keeps a row out of any derive group."""
|
||||
from scribe.services.shape_ledger import (
|
||||
apply_derive_groups, live_rows, propose_for_repo, proposal_summary,
|
||||
)
|
||||
|
||||
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
|
||||
defs = _defs(
|
||||
("a/one.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
|
||||
("a/two.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
|
||||
("b/x.css", "css", "card", ".card {", ".card { padding: 1px }"),
|
||||
("b/y.css", "css", "card", ".card {", ".card { padding: 2px }"),
|
||||
("b/z.css", "css", "card", ".card {", ".card { padding: 3px }"),
|
||||
("c/only.py", "sym", "alone", "def alone():", "def alone():\n return 0"),
|
||||
("c/use.py", "sym", "boot", "def boot():", "def boot():\n return factory()"),
|
||||
)
|
||||
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
|
||||
with _quiet_semantic():
|
||||
await propose_for_repo(owner, pid, REPO, defs)
|
||||
assert await apply_derive_groups(pid) == 5
|
||||
|
||||
rows, total = await list_project_shapes(owner, pid, proposal="derive")
|
||||
assert total == 5
|
||||
groups = {(r.path, r.symbol): r.proposal for r in rows}
|
||||
assert groups[("a/one.py", "slug")]["group"] == groups[("a/two.py", "slug")]["group"]
|
||||
assert groups[("a/one.py", "slug")]["group"].startswith("dup:")
|
||||
assert groups[("b/x.css", "card")] == {"basis": "derive", "score": 3.0, "group": "name:css:card"}
|
||||
rows, _ = await list_project_shapes(owner, pid, proposal="any")
|
||||
assert {r.symbol for r in rows} == {"slug", "card", "boot"} # boot: reference proposal
|
||||
|
||||
summary = proposal_summary(await live_rows(pid))
|
||||
assert summary["proposed"] == 1
|
||||
assert [g["group"] for g in summary["derive_groups"]][0] == "name:css:card"
|
||||
assert summary["derive_groups"][0]["label"] == ".card"
|
||||
assert summary["derive_groups"][0]["size"] == 3
|
||||
|
||||
# One of the css copies gets judged → the group shrinks on the next pass.
|
||||
await classify_shapes(owner, pid, [
|
||||
{"path": "b/z.css", "symbol": "card", "status": "exempt", "reason": "print sheet"},
|
||||
])
|
||||
await apply_derive_groups(pid)
|
||||
rows, _ = await list_project_shapes(owner, pid, proposal="derive")
|
||||
assert {r.symbol for r in rows} == {"slug"} # 2 files < the name floor
|
||||
|
||||
@@ -456,3 +456,54 @@ async def test_unservable_binding_measures_nothing(seeded):
|
||||
await set_binding(uid, "https://github.com/somebody/else.git", other_pid)
|
||||
|
||||
assert await compute_coverage(uid, other_pid, selector=_selector(_tarball(TREE))) is None
|
||||
|
||||
|
||||
# --- #2792: fingerprints and the proposer's readout --------------------------
|
||||
|
||||
|
||||
def test_extract_definitions_fingerprints_each_block():
|
||||
"""The block rule across the language families the extractor knows: a
|
||||
Python def ends at the next top-level statement, a braces/CSS block keeps
|
||||
its closer, and comments/decorators don't move the hash."""
|
||||
from scribe.services.coverage import extract_definitions
|
||||
|
||||
text = (
|
||||
"import os\n\n"
|
||||
"def a(x):\n # comment\n return x + 1\n\n\n"
|
||||
"class B:\n def m(self):\n return 2\n\n"
|
||||
".btn {\n color: red;\n}\n"
|
||||
"export const f = (x) => {\n return x;\n};\n"
|
||||
)
|
||||
defs = {d.name: d for d in extract_definitions(text)}
|
||||
assert set(defs) == {"a", "B", "m", "btn", "f"}
|
||||
assert defs["a"].signature == "def a(x):"
|
||||
assert defs["a"].body.startswith("def a(x):\n # comment\n return x + 1")
|
||||
assert "class B" not in defs["a"].body
|
||||
assert defs["B"].body.rstrip().endswith("return 2")
|
||||
assert defs["btn"].body == ".btn {\n color: red;\n}"
|
||||
assert defs["f"].body == "export const f = (x) => {\n return x;\n};"
|
||||
assert all(len(d.body_sha) == 16 for d in defs.values())
|
||||
# Comment changes don't change what the shape IS; code changes do.
|
||||
again = {d.name: d for d in extract_definitions(text.replace("# comment", "# other"))}
|
||||
assert again["a"].body_sha == defs["a"].body_sha
|
||||
changed = {d.name: d for d in extract_definitions(text.replace("x + 1", "x + 2"))}
|
||||
assert changed["a"].body_sha != defs["a"].body_sha
|
||||
# And the identity view is unchanged for the hook mirror.
|
||||
from scribe.services.coverage import extract_shapes
|
||||
assert extract_shapes(text) == [(d.kind, d.name) for d in extract_definitions(text)]
|
||||
|
||||
|
||||
def test_coverage_line_names_the_proposers_standing():
|
||||
from scribe.services.coverage import coverage_line
|
||||
|
||||
base = {
|
||||
"total": 100, "accounted": 10, "unclassified": 90,
|
||||
"counts": {"canonical": 10, "instance": 0, "variant": 0, "exempt": 0},
|
||||
"computed_at": "2026-08-21T00:00:00+00:00",
|
||||
"largest_gaps": [{"dir": "src", "unclassified": 90, "total": 90}],
|
||||
}
|
||||
assert coverage_line(base).endswith("; 90 unclassified, largest: src")
|
||||
line = coverage_line({**base, "proposed": 40, "derive_groups": [{"group": "a"}, {"group": "b"}]})
|
||||
assert "; 90 unclassified (40 proposed, 2 derive groups), largest: src" in line
|
||||
line = coverage_line({**base, "proposed": 0, "derive_groups": [{"group": "a"}]})
|
||||
assert "(1 derive group)" in line
|
||||
|
||||
@@ -5,6 +5,8 @@ token-free serialisation. The sync pass (step 2) and the classification
|
||||
surface (step 3) grow their tests here; DB-backed behavior lands in the
|
||||
integration lane once there is behavior to exercise.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from scribe.models import Base
|
||||
from scribe.models.code_shape import SHAPE_CLASSIFIERS, SHAPE_STATUSES, CodeShape
|
||||
|
||||
@@ -135,3 +137,142 @@ def test_hook_is_a_server_internal_classifier():
|
||||
|
||||
assert "hook" in SHAPE_CLASSIFIERS
|
||||
assert "hook" not in _CALLER_VIAS
|
||||
|
||||
|
||||
# --- step 6: the mechanical proposer (pure) ---------------------------------
|
||||
|
||||
|
||||
def test_proposal_columns_and_vocabulary_are_pinned():
|
||||
"""Fingerprints + the proposer's standing suggestion live on the row; the
|
||||
basis vocabulary is fixed, with `derive` the odd one out (a group, not a
|
||||
snippet)."""
|
||||
from scribe.models.code_shape import PROPOSAL_BASES
|
||||
|
||||
cols = CodeShape.__table__.c
|
||||
for name in ("signature", "body_sha", "proposed_snippet_id", "proposal_basis",
|
||||
"proposal_score", "proposal_group", "proposed_at", "proposed_sha"):
|
||||
assert name in cols, name
|
||||
fk = next(iter(cols.proposed_snippet_id.foreign_keys))
|
||||
assert fk.ondelete == "SET NULL" and fk.column.table.name == "notes"
|
||||
assert "ix_code_shapes_proposed" in {ix.name for ix in CodeShape.__table__.indexes}
|
||||
assert PROPOSAL_BASES == ("symbol", "text", "reference", "signature", "semantic", "derive")
|
||||
|
||||
|
||||
def test_row_proposal_property_is_one_object_or_none():
|
||||
row = CodeShape(project_id=1, repo_key="r", path="a.py", symbol="f", kind="sym")
|
||||
assert row.proposal is None
|
||||
row.proposed_snippet_id, row.proposal_basis, row.proposal_score = 9, "symbol", 1.0
|
||||
assert row.proposal == {"basis": "symbol", "score": 1.0, "snippet_id": 9}
|
||||
row.proposed_snippet_id = None
|
||||
row.proposal_basis, row.proposal_group, row.proposal_score = "derive", "dup:abc", 3.0
|
||||
assert row.proposal == {"basis": "derive", "score": 3.0, "group": "dup:abc"}
|
||||
|
||||
|
||||
def test_signature_similarity_blanks_the_names():
|
||||
from scribe.services.shape_ledger import signature_similarity as sim
|
||||
|
||||
a = "def move_event(project_id: int, event_id: int, after_id: int | None):"
|
||||
b = "def move_beat(project_id: int, beat_id: int, after_id: int | None):"
|
||||
assert sim(a, "move_event", b, "move_beat") > 0.85
|
||||
assert sim(a, "move_event", "def export_pdf(manuscript, design, fonts):", "export_pdf") < 0.6
|
||||
assert sim("", "x", b, "move_beat") == 0.0
|
||||
# Trivial signatures resemble everything and mean nothing — floored out.
|
||||
assert sim("def helper(x):", "helper", "def make_app():", "make_app") == 0.0
|
||||
|
||||
|
||||
def test_text_containment_is_whitespace_insensitive_with_a_floor():
|
||||
from scribe.services.shape_ledger import text_contains
|
||||
|
||||
code = "const ok = await confirmed({ title: 'Delete?', confirmLabel: 'Delete' });\nif (!ok) return;"
|
||||
body = "async function onDelete() {\n const ok = await confirmed({\n title: 'Delete?',\n confirmLabel: 'Delete'\n });\n if (!ok) return;\n}"
|
||||
assert text_contains(body, code)
|
||||
assert text_contains(code, body)
|
||||
assert not text_contains("x = 1", "x = 1") # below the substance floor
|
||||
|
||||
|
||||
def _canon(sid, kind="sym", symbol="", locations=(), signature="", code=""):
|
||||
from scribe.services.shape_ledger import Canon, _norm_text
|
||||
return Canon(sid, kind, symbol, tuple(locations), signature, _norm_text(code))
|
||||
|
||||
|
||||
def test_match_canon_orders_bases_strongest_first_and_respects_kind():
|
||||
from scribe.services.shape_ledger import match_canon
|
||||
|
||||
confirmed = _canon(
|
||||
7, "sym", "confirmed", [("frontend/src/composables/useConfirm.ts", "confirmed")],
|
||||
"export async function confirmed(opts: ConfirmOptions): Promise<boolean> {",
|
||||
"export async function confirmed(opts: ConfirmOptions): Promise<boolean> { /* singleton */ }",
|
||||
)
|
||||
mover = _canon(
|
||||
8, "sym", "move_beat", [("src/forge/plot.py", "move_beat")],
|
||||
"def move_beat(project_id: int, beat_id: int, after_id: int | None) -> None:",
|
||||
)
|
||||
btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")],
|
||||
".btn-primary {", ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }")
|
||||
canons = [confirmed, mover, btn]
|
||||
|
||||
# symbol: a second `confirmed` defined elsewhere answers to #7 — but the
|
||||
# canon's own location never does (that row is canonical, not a proposal).
|
||||
assert match_canon("sym", "src/other.ts", "confirmed", "function confirmed() {", "", canons) == (7, "symbol", 1.0)
|
||||
assert match_canon("sym", "frontend/src/composables/useConfirm.ts", "confirmed",
|
||||
"export async function confirmed(", "", canons) is None
|
||||
# reference: a call site of the canon.
|
||||
body = "async function onTrash() {\n const ok = await confirmed({ title: 'x' });\n if (!ok) return;\n}"
|
||||
assert match_canon("sym", "c.vue", "onTrash", "async function onTrash() {", body, canons) == (7, "reference", 0.9)
|
||||
# signature: the family shape, names blanked.
|
||||
hit = match_canon("sym", "src/forge/timeline.py", "move_event",
|
||||
"def move_event(project_id: int, event_id: int, after_id: int | None) -> None:",
|
||||
" pass", canons)
|
||||
assert hit and hit[0] == 8 and hit[1] == "signature" and hit[2] >= 0.8
|
||||
# text: the canon's code contains the shape's body (a css copy), kind-matched —
|
||||
# the same text as a `sym` shape matches no css canon.
|
||||
css_body = ".btn-primary { color: var(--action-primary); padding: 4px 8px; border-radius: 4px; }"
|
||||
assert match_canon("css", "web/other.css", "btn-big", ".btn-big {", css_body, canons) == (9, "text", 0.95)
|
||||
assert match_canon("sym", "web/other.css", "btn-big", ".btn-big {", css_body, canons) is None
|
||||
# nothing in play
|
||||
assert match_canon("sym", "x.py", "unrelated", "def unrelated(a, b, c, d, e):", "return 1", canons) is None
|
||||
|
||||
|
||||
def test_match_canon_symbol_beats_everything_including_css_copies():
|
||||
"""The previous test's css `btn-primary`-elsewhere case, stated plainly:
|
||||
a second definition of the canon's own name is the symbol basis."""
|
||||
from scribe.services.shape_ledger import match_canon
|
||||
btn = _canon(9, "css", ".btn-primary", [("web/buttons.css", ".btn-primary")], ".btn-primary {", ".btn-primary { color: red; padding: 4px 8px; border-radius: 4px; }")
|
||||
assert match_canon("css", "web/other.css", "btn-primary", ".btn-primary {", ".btn-primary { color: blue }", [btn]) == (9, "symbol", 1.0)
|
||||
|
||||
|
||||
def test_derive_groups_copy_before_name_with_floors():
|
||||
from scribe.services.shape_ledger import derive_groups
|
||||
|
||||
rows = [
|
||||
("a.py", "sym", "helper", "sha1"), ("b.py", "sym", "helper", "sha1"), # identical copies
|
||||
("c.py", "sym", "helper", "sha9"), # same name, 3rd file
|
||||
("d.css", "css", "btn", "s1"), ("e.css", "css", "btn", "s2"), ("f.css", "css", "btn", "s3"),
|
||||
("g.py", "sym", "main", "s4"), ("h.py", "sym", "main", "s5"), # only 2 files → no name group
|
||||
("i.py", "sym", "one", "s6"),
|
||||
]
|
||||
g = derive_groups(rows)
|
||||
assert g[("a.py", "sym", "helper")] == "dup:sha1" == g[("b.py", "sym", "helper")]
|
||||
assert g[("c.py", "sym", "helper")] == "name:sym:helper"
|
||||
assert g[("d.css", "css", "btn")] == "name:css:btn"
|
||||
assert ("g.py", "sym", "main") not in g
|
||||
assert ("i.py", "sym", "one") not in g
|
||||
|
||||
|
||||
def test_confirm_requires_a_named_scope():
|
||||
import asyncio
|
||||
|
||||
from scribe.services.shape_ledger import confirm_proposals
|
||||
|
||||
with pytest.raises(ValueError) as err:
|
||||
asyncio.run(confirm_proposals(1, 2))
|
||||
assert "name what you reviewed" in str(err.value)
|
||||
|
||||
|
||||
def test_proposer_tools_are_mounted():
|
||||
from scribe.mcp.server import build_mcp_server
|
||||
|
||||
mcp = build_mcp_server()
|
||||
assert mcp._tool_manager.get_tool("confirm_shape_proposals") is not None
|
||||
tool = mcp._tool_manager.get_tool("list_shapes")
|
||||
assert "proposal" in tool.parameters.get("properties", {})
|
||||
|
||||
Reference in New Issue
Block a user