feat(ledger): mechanical proposer — every refresh proposes instances against canon and groups derive-first candidates; agents confirm in batches (#2792, milestone 294 step 6)

Shapes now carry a content fingerprint (signature + whitespace/comment-
insensitive body_sha; migration 0080) and the proposer runs inside the
coverage refresh, the one moment bodies exist: symbol elsewhere → textual
containment → body references the canon → signature resemblance → semantic
(capped per refresh, unreached rows stay unexamined for the next). A hit is
a proposal on the row (proposed_snippet_id/basis/score), never a
classification; rows with no canon hit group by the derive-first rule
(identical body in ≥2 places, same name in ≥3 files) as proposal_basis=
derive + a group key. list_shapes(proposal=any|canon|derive|<basis>) is the
queue; confirm_shape_proposals(project_id, snippet_id|path|basis) confirms
in batches as agent instances; any classify_shapes/hook stamp retires the
proposal. Readout carries proposed + derive_groups (line, payload, card).
Plugin 0.1.35 (skill: the machine proposes, judgment classifies).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/scribe/services/coverage.py

@@ -25,12 +25,14 @@ only ever reads the cache.
 """
 from __future__ import annotations
 
+import hashlib
 import io
 import json
 import logging
 import posixpath
 import re
 import tarfile
+from typing import NamedTuple
 from datetime import datetime, timedelta, timezone
 
 from scribe.services.forge import ForgeSelector, get_forges
@@ -91,6 +93,104 @@ _ARROW_RE = re.compile(
 )
 
 
+class Definition(NamedTuple):
+    """One extracted definition with its content fingerprint (#2792).
+
+    `signature` is the definition line itself; `body_sha` hashes the block
+    whitespace- and comment-insensitively; `body` is the block's text, held
+    only for the duration of a refresh (the proposer matches on it) and
+    never stored.
+    """
+
+    kind: str
+    name: str
+    signature: str
+    body_sha: str
+    body: str
+
+
+def _definition_on(raw: str) -> tuple[str, str] | None:
+    """The (kind, name) this one line defines, or None. First match wins —
+    the same order the hook's awk program tries."""
+    m = _CSS_RE.match(raw)
+    if m:
+        return ("css", m.group(1))
+    line = _MODIFIERS_RE.sub("", raw.lstrip())
+    if m := _GO_METHOD_RE.match(line):
+        return ("sym", m.group(1))
+    if m := _KEYWORD_RE.match(line):
+        name = m.group(1)
+        if name.startswith("__") and name.endswith("__"):
+            return None
+        return ("sym", name)
+    if m := _ARROW_RE.match(line):
+        return ("sym", m.group(1))
+    return None
+
+
+# A definition's block runs from its line until the next non-blank line at
+# its own indentation or shallower that is not a closer — so a Python def ends
+# at the next top-level statement, a braces block keeps its `}`, a CSS rule
+# keeps its `}`. Capped so a generated monolith can't make one shape's
+# fingerprint cover the file.
+_BLOCK_CAP = 120
+_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);")
+# Lines that don't change what a shape IS: comments and decorators. Dropped
+# from the fingerprint so touching a comment above the next function doesn't
+# read as this one's body changing.
+_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "<!--", "-->")
+_SIGNATURE_CAP = 300
+
+
+def _indent(line: str) -> int:
+    return len(line) - len(line.lstrip())
+
+
+def _block_sha(lines: list[str]) -> str:
+    kept = [
+        " ".join(ln.split())
+        for ln in lines
+        if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES)
+    ]
+    return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16]
+
+
+def extract_definitions(text: str) -> list[Definition]:
+    """Every definition this text makes, with signature + fingerprint.
+
+    Duplicate (kind, name) within one text collapse to the first — the
+    ledger's identity is per file, so a second definition of the same name
+    (an overload, a re-declaration) is the same shape to it.
+    """
+    lines = text.splitlines()
+    starts: list[tuple[int, str, str]] = []
+    for i, raw in enumerate(lines):
+        hit = _definition_on(raw)
+        if hit:
+            starts.append((i, hit[0], hit[1]))
+    seen: set[tuple[str, str]] = set()
+    out: list[Definition] = []
+    for i, kind, name in starts:
+        if (kind, name) in seen:
+            continue
+        seen.add((kind, name))
+        base = _indent(lines[i])
+        end = min(len(lines), i + _BLOCK_CAP)
+        for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)):
+            ln = lines[j]
+            if not ln.strip():
+                continue
+            if _indent(ln) <= base and ln.strip() not in _CLOSERS:
+                end = j
+                break
+        block = lines[i:end]
+        out.append(Definition(
+            kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(block),
+            "\n".join(block),
+        ))
+    return out
+
+
 def extract_shapes(text: str) -> list[tuple[str, str]]:
     """Every (kind, name) this text DEFINES — kind is "css" or "sym".
 
@@ -98,29 +198,7 @@ def extract_shapes(text: str) -> list[tuple[str, str]]:
     line, dunders are skipped (every class defines __init__ — guaranteed
     noise), duplicates within one text count once.
     """
-    seen: set[tuple[str, str]] = set()
-    out: list[tuple[str, str]] = []
-    for raw in text.splitlines():
-        m = _CSS_RE.match(raw)
-        if m:
-            shape = ("css", m.group(1))
-        else:
-            line = _MODIFIERS_RE.sub("", raw.lstrip())
-            if m := _GO_METHOD_RE.match(line):
-                shape = ("sym", m.group(1))
-            elif m := _KEYWORD_RE.match(line):
-                name = m.group(1)
-                if name.startswith("__") and name.endswith("__"):
-                    continue
-                shape = ("sym", name)
-            elif m := _ARROW_RE.match(line):
-                shape = ("sym", m.group(1))
-            else:
-                continue
-        if shape not in seen:
-            seen.add(shape)
-            out.append(shape)
-    return out
+    return [(d.kind, d.name) for d in extract_definitions(text)]
 
 
 def scannable(path: str) -> bool:
@@ -131,14 +209,33 @@ def scannable(path: str) -> bool:
     return not path.lower().endswith(_SKIP_SUFFIXES)
 
 
+class ArchiveShape(NamedTuple):
+    """A definition located in a repo archive — what the sync upserts and
+    the proposer matches. The leading (path, kind, name) triple is the
+    ledger identity; the rest is the fingerprint and the transient body."""
+
+    path: str
+    kind: str
+    name: str
+    signature: str
+    body_sha: str
+    body: str
+
+
 def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
-    """(path, kind, name) for every definition in a repo tarball.
+    """(path, kind, name) for every definition in a repo tarball — the
+    identity view of definitions_from_archive."""
+    return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)]
+
+
+def definitions_from_archive(blob: bytes) -> list[ArchiveShape]:
+    """Every definition in a repo tarball, with its fingerprint and body.
 
     Forge archives wrap content in a single top-level directory (repo-ref/);
     that component is stripped so paths match recorded snippet locations,
     which are repo-relative. Non-UTF-8 files are binaries and skipped.
     """
-    shapes: list[tuple[str, str, str]] = []
+    shapes: list[ArchiveShape] = []
     with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
         for member in tar:
             if not member.isfile() or "/" not in member.name:
@@ -153,7 +250,10 @@ def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
                 text = handle.read().decode("utf-8")
             except UnicodeDecodeError:
                 continue
-            shapes.extend((path, kind, name) for kind, name in extract_shapes(text))
+            shapes.extend(
+                ArchiveShape(path, d.kind, d.name, d.signature, d.body_sha, d.body)
+                for d in extract_definitions(text)
+            )
     return shapes
 
 
@@ -253,13 +353,17 @@ async def compute_coverage(
 
     served: list[tuple[str, str]] = []
     recorded = await _recorded_locations(user_id, project_id)
+    # The proposer's canon catalog, read once per refresh and shared across
+    # the project's repos (#2792).
+    canons = None
+    proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0}
     for key in await keys_for_project(user_id, project_id):
         hit = selector.resolve(key)
         if hit is None:
             continue  # bound to a host no connection serves
         forge, api_repo = hit
         ref = await forge.default_branch(api_repo)
-        shapes = shapes_from_archive(await forge.archive(api_repo, ref))
+        definitions = definitions_from_archive(await forge.archive(api_repo, ref))
         # The head commit is provenance sugar on the ledger rows; failing to
         # learn it must not fail the sync — the ref names the point well
         # enough and the row timestamps carry the when.
@@ -268,13 +372,31 @@ async def compute_coverage(
         except ForgeError:
             marker = ref
         await shape_ledger.sync_repo_shapes(
-            project_id, key, shapes, seen_marker=marker
+            project_id, key, definitions, seen_marker=marker
         )
         served.append((key, ref))
+        # Propose while the bodies are in hand — the one moment they exist.
+        # Canonical marking below only touches rows the proposer leaves
+        # alone (a canon's own location never gets a proposal), so the order
+        # is immaterial; the proposer must not be able to fail the refresh.
+        try:
+            if canons is None:
+                canons = await shape_ledger.canon_catalog(user_id)
+            stats = await shape_ledger.propose_for_repo(
+                user_id, project_id, key, definitions, canons=canons
+            )
+            for k in proposer_stats:
+                proposer_stats[k] += stats.get(k, 0)
+        except Exception:
+            logger.warning("shape proposer failed for %s", key, exc_info=True)
     if not served:
         return None
 
     await shape_ledger.mark_canonicals(project_id, recorded)
+    try:
+        await shape_ledger.apply_derive_groups(project_id)
+    except Exception:
+        logger.warning("derive-first grouping failed", exc_info=True)
 
     # Project-wide readout, deliberately wider than this walk: a second bound
     # repo that was unreachable today still has live rows, and they count.
@@ -290,11 +412,17 @@ async def compute_coverage(
         agg["accounted"] += row.status != "unclassified"
 
     unclassified = counts.pop("unclassified")
+    proposals = shape_ledger.proposal_summary(rows)
     return {
         "total": len(rows),
         "accounted": len(rows) - unclassified,
         "unclassified": unclassified,
         "counts": counts,
+        # The proposer's standing (#2792): canon proposals awaiting a
+        # confirm, the largest derive-first groups, and what this refresh did.
+        "proposed": proposals["proposed"],
+        "derive_groups": proposals["derive_groups"],
+        "proposer": proposer_stats,
         # Honesty flag, not decoration: every surface that shows the number
         # is expected to carry it through.
         "estimate": True,
@@ -438,6 +566,14 @@ def coverage_line(coverage: dict) -> str:
     unclassified = coverage.get("unclassified", 0)
     if unclassified:
         line += f"; {unclassified} unclassified"
+        standing = []
+        if coverage.get("proposed"):
+            standing.append(f"{coverage['proposed']} proposed")
+        n_groups = len(coverage.get("derive_groups") or [])
+        if n_groups:
+            standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}")
+        if standing:
+            line += f" ({', '.join(standing)})"
         gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
         if gaps:
             line += ", largest: " + ", ".join(gaps)