feat(ledger): coverage refresh feeds the shape ledger; the readout inverts to accounting (#2788, milestone 294 step 2)

compute_coverage is now the ledger's sync point: every walk upserts the
extracted shapes (new → unclassified, the todo state; surviving → last-seen
bump; vanished → stamped, kept as history), re-files judgments whose snippet
target went away, and mechanically stamps snippet reference locations as
canonical — the one always-safe rule, self-healing only for its own stamps
(an agent's judgment is never unwound by machinery).

The covering predicate moves to shape_ledger.location_covers as the single
home (match_shapes retired with its consumer); coverage's payload and line
invert from 'N/M shapes recorded' to shape ACCOUNTING per note 2786:
accounted/total with a canonical·instance·variant·exempt breakdown, and
unclassified — THE todo — with its largest directories. Cache key bumps to
v2 so pre-ledger blobs honestly read 'not measured yet' instead of rendering
in a shape no longer spoken.

Readout is deliberately project-wide (all repos' live rows), while the walk
serves whichever repos the owner's keyring reaches this refresh.

Integration tests pin the new contract: rows for every extracted shape,
mechanical canonical stamps carrying snippet ids, idempotent recompute,
agent judgments surviving recompute AND vanish/return, vanished rows leaving
the readout but keeping their history.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/scribe/services/coverage.py

@@ -41,7 +41,10 @@ logger = logging.getLogger(__name__)
 
 # Cache key in the settings KV, on the project OWNER's user_id — the same
 # channel the scheduler's last-run summary uses for machine-written state.
-_CACHE_KEY_PREFIX = "pattern_coverage_"
+# v2 suffix with #2788: the payload shape inverted (accounted/unclassified);
+# pre-ledger blobs under the old key simply stop being found, so the card
+# honestly reads "not measured yet" until the first ledger-era refresh.
+_CACHE_KEY_PREFIX = "pattern_coverage_v2_"
 
 # Files whose content can't hold definitions — the hook's skip list, verbatim,
 # plus sourcemaps (which are JSON in a trenchcoat).
@@ -157,77 +160,42 @@ def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
 # --- matching shapes against recorded locations ------------------------------
 
 
-def _norm_symbol(kind_or_symbol: str) -> str:
-    # CSS shapes and recorded CSS symbols may or may not carry the leading
-    # dot; compare without it so ".btn-primary" and "btn-primary" agree.
-    return kind_or_symbol.lstrip(".").strip()
-
-
-def _location_covers(loc_path: str, loc_symbol: str, path: str, name: str) -> bool:
-    if _norm_symbol(loc_symbol) != _norm_symbol(name):
-        return False
-    if not loc_path:
-        # Symbol-only record: the symbol match is all the claim there is.
-        return True
-    # The drift check's location semantics, not a second copy of them: exact
-    # file, or the recorded path is a directory the file lives under.
-    from scribe.services.snippets import _path_touches
-
-    return _path_touches(loc_path, path)
-
-
-def match_shapes(
-    shapes: list[tuple[str, str, str]],
-    recorded: list[tuple[str, str]],
-) -> list[tuple[str, str, str, bool]]:
-    """Each shape with whether some recorded (path, symbol) location covers it.
-
-    Symbol-less recorded locations never cover a shape — a whole-file record
-    makes no claim about any particular definition inside it. The recorded
-    repo NAME is deliberately not consulted: it is free-form ("Scribe") and
-    the project binding already did the scoping; on a project binding several
-    repos this can over-credit a same-named symbol, which the estimate label
-    owns.
-    """
-    usable = [(p, s) for p, s in recorded if (s or "").strip()]
-    return [
-        (
-            path,
-            kind,
-            name,
-            any(_location_covers(lp, ls, path, name) for lp, ls in usable),
-        )
-        for path, kind, name in shapes
-    ]
+# The covering predicate (location_covers) lives in services/shape_ledger.py
+# since #2788 — the ledger's canonical marking and this module's readout are
+# two consumers of ONE doctrine, and the ledger is its home.
 
 
 def largest_gaps(
-    matched: list[tuple[str, str, str, bool]], *, top: int = 3
+    accounted: list[tuple[str, str, str, bool]], *, top: int = 3
 ) -> list[dict]:
-    """The directories with the most uncovered shapes — where a backlog
-    session should start, named the way the repo names them."""
+    """The directories with the most unclassified shapes — where a
+    classification session should start, named the way the repo names them."""
     by_dir: dict[str, dict[str, int]] = {}
-    for path, _kind, _name, covered in matched:
+    for path, _kind, _name, is_accounted in accounted:
         d = posixpath.dirname(path) or "(root)"
-        row = by_dir.setdefault(d, {"total": 0, "uncovered": 0})
+        row = by_dir.setdefault(d, {"total": 0, "unclassified": 0})
         row["total"] += 1
-        if not covered:
-            row["uncovered"] += 1
+        if not is_accounted:
+            row["unclassified"] += 1
     ranked = sorted(
-        by_dir.items(), key=lambda kv: (-kv[1]["uncovered"], kv[0])
+        by_dir.items(), key=lambda kv: (-kv[1]["unclassified"], kv[0])
     )
     return [
-        {"dir": d, "uncovered": row["uncovered"], "total": row["total"]}
+        {"dir": d, "unclassified": row["unclassified"], "total": row["total"]}
         for d, row in ranked[:top]
-        if row["uncovered"]
+        if row["unclassified"]
     ]
 
 
 # --- compute, cache, surface -------------------------------------------------
 
 
-async def _recorded_locations(user_id: int, project_id: int) -> list[tuple[str, str]]:
-    """(path, symbol) for every location of every live snippet in a project."""
+async def _recorded_locations(
+    user_id: int, project_id: int
+) -> list[tuple[int, str, str]]:
+    """(snippet_note_id, path, symbol) for every location of every live
+    snippet in a project — the canonical-marking input (#2788): the id is what
+    lets a ledger row point back at the snippet it references."""
     from sqlalchemy import select
 
     from scribe.models import async_session
@@ -244,34 +212,46 @@ async def _recorded_locations(user_id: int, project_id: int) -> list[tuple[str,
             )
         )
         notes = list(rows.scalars().all())
-    out: list[tuple[str, str]] = []
+    out: list[tuple[int, str, str]] = []
     for note in notes:
         for loc in snippet_fields(note).get("locations") or []:
-            out.append((loc.get("path") or "", loc.get("symbol") or ""))
+            out.append(
+                (int(note.id), loc.get("path") or "", loc.get("symbol") or "")
+            )
     return out
 
 
 async def compute_coverage(
     user_id: int, project_id: int, *, selector: ForgeSelector | None = None
 ) -> dict | None:
-    """Measure a project's pattern-library coverage against its bound repos.
+    """Sync the shape ledger from the bound repos and read the accounting.
+
+    Since #2788 this is the ledger's sync point, not just a measurement:
+    every walk upserts the extracted shapes (new → unclassified, vanished →
+    stamped), re-stamps snippet reference locations as canonical, and then
+    reports the ACCOUNTING — how many shapes carry a classification at all —
+    rather than the old "has a snippet" fraction. Unclassified is the todo
+    (note 2786).
 
     None means "nothing to measure" — the owner's keyring serves none of the
     project's bound repos (#2778). That is the ordinary state for a
-    forge-less user and every caller treats it as silence, not failure.
-    Forge errors (unreachable, bad token) RAISE — the two callers are a
-    refresh button and a background task, and both want to know.
+    forge-less user and every caller treats it as silence, not failure; the
+    ledger is untouched in that case. Forge errors (unreachable, bad token)
+    RAISE — the two callers are a refresh button and a background task, and
+    both want to know.
 
     ``user_id`` is the project OWNER's id: the cache lives there, and the
     keyring resolved here must be the same one every other read uses.
     """
+    from scribe.services import shape_ledger
+    from scribe.services.forge import ForgeError
+
     if selector is None:
         selector = await get_forges(user_id, project_id)
     if not selector.configured:
         return None
 
-    repos: list[dict] = []
-    matched_all: list[tuple[str, str, str, bool]] = []
+    served: list[tuple[str, str]] = []
     recorded = await _recorded_locations(user_id, project_id)
     for key in await keys_for_project(user_id, project_id):
         hit = selector.resolve(key)
@@ -280,26 +260,54 @@ async def compute_coverage(
         forge, api_repo = hit
         ref = await forge.default_branch(api_repo)
         shapes = shapes_from_archive(await forge.archive(api_repo, ref))
-        matched = match_shapes(shapes, recorded)
-        matched_all.extend(matched)
-        repos.append({
-            "repo": key,
-            "ref": ref,
-            "total": len(matched),
-            "recorded": sum(1 for *_x, covered in matched if covered),
-        })
-    if not repos:
+        # The head commit is provenance sugar on the ledger rows; failing to
+        # learn it must not fail the sync — the ref names the point well
+        # enough and the row timestamps carry the when.
+        try:
+            marker = await forge.latest_commit(api_repo, "", ref) or ref
+        except ForgeError:
+            marker = ref
+        await shape_ledger.sync_repo_shapes(
+            project_id, key, shapes, seen_marker=marker
+        )
+        served.append((key, ref))
+    if not served:
         return None
 
+    await shape_ledger.mark_canonicals(project_id, recorded)
+
+    # Project-wide readout, deliberately wider than this walk: a second bound
+    # repo that was unreachable today still has live rows, and they count.
+    rows = await shape_ledger.live_rows(project_id)
+    counts = {"canonical": 0, "instance": 0, "variant": 0, "exempt": 0,
+              "unclassified": 0}
+    for row in rows:
+        counts[row.status] = counts.get(row.status, 0) + 1
+    by_repo: dict[str, dict[str, int]] = {}
+    for row in rows:
+        agg = by_repo.setdefault(row.repo_key, {"total": 0, "accounted": 0})
+        agg["total"] += 1
+        agg["accounted"] += row.status != "unclassified"
+
+    unclassified = counts.pop("unclassified")
     return {
-        "total": len(matched_all),
-        "recorded": sum(1 for *_x, covered in matched_all if covered),
+        "total": len(rows),
+        "accounted": len(rows) - unclassified,
+        "unclassified": unclassified,
+        "counts": counts,
         # Honesty flag, not decoration: every surface that shows the number
         # is expected to carry it through.
         "estimate": True,
         "computed_at": datetime.now(timezone.utc).isoformat(),
-        "repos": repos,
-        "largest_gaps": largest_gaps(matched_all),
+        "repos": [
+            {"repo": key, "ref": ref,
+             **by_repo.get(key, {"total": 0, "accounted": 0})}
+            for key, ref in served
+        ],
+        "largest_gaps": largest_gaps([
+            (r.path, r.kind, r.symbol, r.status != "unclassified")
+            for r in rows
+        ]),
     }
 
 
@@ -330,12 +338,23 @@ async def cached_coverage(user_id: int, project_id: int) -> dict | None:
 def coverage_line(coverage: dict) -> str:
     """The one-line evidence-carrying summary enter_project surfaces."""
     day = (coverage.get("computed_at") or "")[:10]
-    line = (
-        f"pattern-library coverage: {coverage.get('recorded', 0)}"
-        f"/{coverage.get('total', 0)} shapes recorded"
-        f" (estimate{', computed ' + day if day else ''})"
+    counts = coverage.get("counts") or {}
+    breakdown = " · ".join(
+        f"{counts[k]} {k}"
+        for k in ("canonical", "instance", "variant", "exempt")
+        if counts.get(k)
     )
-    gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
-    if gaps:
-        line += "; largest gaps: " + ", ".join(gaps)
+    line = (
+        f"shape accounting: {coverage.get('accounted', 0)}"
+        f"/{coverage.get('total', 0)} shapes accounted for"
+    )
+    if breakdown:
+        line += f" — {breakdown}"
+    line += f" (estimate{', computed ' + day if day else ''})"
+    unclassified = coverage.get("unclassified", 0)
+    if unclassified:
+        line += f"; {unclassified} unclassified"
+        gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
+        if gaps:
+            line += ", largest: " + ", ".join(gaps)
     return line