feat(family): the shape ledger judges against family ideas, across languages (milestone 463 step 5, #4991)

- classify_shapes takes idea_id: the shape is judged against the idea's
  reference in its language, or its first.
- A shape classified against a canon idea's reference moves the project's
  adoption row (instance -> adopted, variant -> variant). Withdrawing the
  shapes that gave an answer returns it to unassessed. This runs on
  classify_shapes, the sweep, confirm_proposals and the coverage refresh.
- assess and undo refuse an answer the shapes contradict. A conflict
  resolution re-judges the variant shapes on both sides.
- The proposer's family arm offers a canon idea's reference on a shared
  platform, in any language. It proposes only when the reference is the top
  hit over every readable snippet and scores at least 0.70. The pairs this
  was measured on are recorded beside _FAMILY_FLOOR. The proposer is now v6.
- The matrix cell shows the code that answers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/scribe/services/family_adoption.py

@@ -43,13 +43,26 @@ applies wins, and every ground above it must be said not to:
 The losing side's reasoning is folded into the idea's note — as a trap, an
 alternative, or the branch for its condition — and is never dropped. The
 substance changed, so the version moves.
+
+SHAPES (step 5). The shape ledger answers too: a shape classified against an
+idea's reference — in any language — is the project's code saying `adopted`
+(an instance) or `variant` (with the shape's reason). The two ledgers never
+disagree: a classification moves the row, an assessment or undo the shapes
+contradict is refused, an answer the shapes gave is withdrawn with them, and
+a conflict resolution re-judges the variant shapes on both sides. The shape
+proposer offers an idea's reference across projects and languages by
+meaning; its rule and the measurement behind it sit above
+shape_ledger._FAMILY_FLOOR.
 """
 from __future__ import annotations
 
 import logging
+from typing import Iterable
+
 from sqlalchemy import func, select
 
 from scribe.models import async_session
+from scribe.models.code_shape import CodeShape
 from scribe.models.family import (
     FamilyAdoption, FamilyDecision, FamilyIdea, FamilyIdeaPlatform, FamilyIdeaReference,
     Platform, ProjectPlatform,
@@ -281,6 +294,224 @@ async def _project_languages(session, project_id: int) -> list[str]:
     return [r[0] for r in rows]
 
 
+# --- the shape ledger as evidence (milestone 463 step 5) ----------------------------
+#
+# A shape classified against an idea's reference IS an answer to the idea: an
+# `instance` — or the reference's own `canonical` location — says the project
+# does it; a `variant` says it departs, for the reason the shape carries. The
+# two ledgers must never disagree, and the shapes are the stronger evidence
+# (they name the code), so:
+#
+#   - classifying a shape moves the adoption row to what the shapes say,
+#     decided_via "system", the shapes as evidence;
+#   - an assessment or an undo that would contradict the shapes is refused —
+#     reclassify the shapes if they are wrong;
+#   - an answer the shape ledger gave is withdrawn to `unassessed` when the
+#     shapes that gave it are withdrawn. An answer given on other evidence is
+#     left alone: no shape is not the same as `owed`.
+
+SHAPE_SOURCE = "shape_ledger"
+_SHAPE_ADOPTS = ("canonical", "instance")
+
+# Which reference a shape is judged against when it is classified against an
+# idea: the one in the shape's language when there is one.
+_EXTS_BY_LANGUAGE = {
+    "python": (".py", ".pyi"), "go": (".go",), "kotlin": (".kt", ".kts"),
+    "java": (".java",), "dart": (".dart",), "swift": (".swift",), "rust": (".rs",),
+    "typescript": (".ts", ".tsx"), "javascript": (".js", ".jsx", ".mjs", ".cjs"),
+    "vue": (".vue",), "svelte": (".svelte",), "css": (".css",), "scss": (".scss",),
+    "bash": (".sh", ".bash"), "sh": (".sh",), "sql": (".sql",),
+}
+
+
+def path_speaks(path: str, language: str) -> bool:
+    """Is a file at ``path`` written in ``language`` (a snippet's recorded one)?"""
+    exts = _EXTS_BY_LANGUAGE.get((language or "").strip().lower())
+    return bool(exts) and (path or "").strip().lower().endswith(exts)
+
+
+def shape_outcome(statuses: Iterable[str]) -> str | None:
+    """What a project's shapes say about an idea: `adopted` when any shape is
+    an instance of (or is) a reference, `variant` when the only ones are
+    variants, None when no shape speaks. None is silence, never `owed`."""
+    seen = set(statuses)
+    if seen & set(_SHAPE_ADOPTS):
+        return "adopted"
+    if "variant" in seen:
+        return "variant"
+    return None
+
+
+def shapes_say(rows) -> dict | None:
+    """The answer a project's evidence rows give — outcome, reason, evidence
+    lines and the shapes themselves — or None when no shape speaks. The
+    `adopted` reason is fixed text, so classifying one more instance does not
+    log a new decision; a `variant` carries the first variant shape's why."""
+    outcome = shape_outcome(r.status for r in rows)
+    if outcome is None:
+        return None
+    speaking = [r for r in rows if (r.status in _SHAPE_ADOPTS) == (outcome == "adopted")]
+    if outcome == "adopted":
+        reason = "the shape ledger classifies this project's code as the idea's reference implementation"
+    else:
+        first = speaking[0]
+        reason = (f"the shape ledger classifies {first.path}::{first.symbol} as a variant "
+                  f"of #{first.snippet_id}: {(first.reason or '').strip()}")
+    return {
+        "outcome": outcome,
+        "reason": reason,
+        "evidence": [f"{r.path}::{r.symbol}" for r in speaking],
+        "shapes": [{"id": r.id, "path": r.path, "symbol": r.symbol, "status": r.status,
+                    "snippet_id": r.snippet_id} for r in speaking],
+    }
+
+
+async def _shape_evidence(session, project_id: int, idea_id: int) -> list[CodeShape]:
+    """The project's live shapes judged against any of the idea's references."""
+    refs = select(FamilyIdeaReference.snippet_id).where(FamilyIdeaReference.idea_id == idea_id)
+    return list((await session.execute(
+        select(CodeShape).where(
+            CodeShape.project_id == project_id,
+            CodeShape.vanished_at.is_(None),
+            CodeShape.snippet_id.in_(refs),
+            CodeShape.status.in_(_SHAPE_ADOPTS + ("variant",)),
+        ).order_by(CodeShape.path, CodeShape.symbol)
+    )).scalars().all())
+
+
+def _contradiction(outcome: str, said: dict | None, idea_id: int) -> str | None:
+    if said is None or outcome == said["outcome"]:
+        return None
+    named = ", ".join(said["evidence"][:3]) + (" …" if len(said["evidence"]) > 3 else "")
+    return (
+        f"the shape ledger already answers #{idea_id} in this project as {said['outcome']} "
+        f"({named}). The two ledgers never disagree: reclassify those shapes "
+        "(classify_shapes) if they are wrong, and the answer follows")
+
+
+async def reference_for(user_id: int, idea_id: int, path: str) -> int:
+    """The reference snippet a shape at ``path`` is judged against when it is
+    classified against a family idea: the one in the path's language when
+    there is one, else the first. An idea is shared across languages, so a
+    Python shape can be an instance of an idea whose only reference is Go.
+    Raises ValueError when the idea is not a family idea the caller can read
+    or has no reference they can read."""
+    if not await access.can_read_note(user_id, idea_id):
+        raise ValueError(f"#{idea_id} is not a family idea you can read")
+    async with async_session() as session:
+        if await session.get(FamilyIdea, idea_id) is None:
+            raise ValueError(f"#{idea_id} is not a family idea")
+        refs = await _references(session, idea_id)
+    readable = [r for r in refs if await access.can_read_note(user_id, r["id"])]
+    if not readable:
+        raise ValueError(
+            f"family idea #{idea_id} has no reference implementation you can read — "
+            "set_family_references first, or classify against a snippet_id")
+    for r in readable:
+        if path_speaks(path, r["language"]):
+            return r["id"]
+    return readable[0]["id"]
+
+
+async def family_reference_ids(project_id: int) -> set[int]:
+    """The reference snippets of every canon idea that reaches the project —
+    what the shape proposer may match across projects and languages."""
+    async with async_session() as session:
+        return set((await session.execute(
+            select(FamilyIdeaReference.snippet_id)
+            .join(FamilyIdea, FamilyIdea.note_id == FamilyIdeaReference.idea_id)
+            .join(FamilyIdeaPlatform, FamilyIdeaPlatform.note_id == FamilyIdea.note_id)
+            .join(ProjectPlatform, ProjectPlatform.platform_id == FamilyIdeaPlatform.platform_id)
+            .where(
+                FamilyIdea.status == "canon",
+                ProjectPlatform.project_id == project_id,
+                ProjectPlatform.state.in_(family_svc.MEMBER_STATES),
+            )
+        )).scalars().all())
+
+
+async def _latest_source(session, project_id: int, idea_id: int) -> str:
+    latest = (await session.execute(
+        select(FamilyDecision).where(
+            FamilyDecision.idea_id == idea_id, FamilyDecision.project_id == project_id)
+        .order_by(FamilyDecision.id.desc()).limit(1)
+    )).scalars().first()
+    return str(((latest.evidence if latest is not None else None) or {}).get("source") or "")
+
+
+async def _withdraw_shape_answer(user_id: int, project_id: int, idea_id: int) -> FamilyDecision:
+    reason = ("the shapes that answered this idea were withdrawn from the shape ledger; "
+              "it is unassessed again")
+    async with async_session() as session:
+        row = await _row(session, project_id, idea_id)
+        before = _row_state(row)
+        row.status = "unassessed"
+        row.reason = None
+        row.canon_version = None
+        row.assessed_at = None
+        row.decided_via = None
+        row.updated_at = family_svc._now()
+        decision = family_svc._log(
+            session, idea_id=idea_id, project_id=project_id, action="assess",
+            reason=reason, before=before, after=_row_state(row),
+            evidence={"source": SHAPE_SOURCE, "shapes": []}, precedent_ids=None,
+            decided_via="system", user_id=user_id,
+        )
+        await session.commit()
+        await session.refresh(decision)
+        row_id = row.id
+    await _sync_owed_task(user_id, row_id, reason=reason)
+    return decision
+
+
+async def sync_from_shapes(user_id: int, project_id: int,
+                           snippet_ids: Iterable[int] | None = None) -> list[dict]:
+    """Bring the project's adoption rows into line with its shape ledger, for
+    the canon ideas whose references are among ``snippet_ids`` — or every
+    canon idea that reaches the project when None (the coverage refresh,
+    which also sees shapes vanish).
+
+    A row that already gives the shapes' outcome at the current version is
+    left alone, whatever evidence it was given on. Returns one entry per
+    answer that moved: {idea_id, status, decision_id}."""
+    query = select(FamilyIdea.note_id).where(FamilyIdea.status == "canon")
+    if snippet_ids is not None:
+        wanted = {int(s) for s in snippet_ids if s}
+        if not wanted:
+            return []
+        query = query.where(FamilyIdea.note_id.in_(
+            select(FamilyIdeaReference.idea_id).where(FamilyIdeaReference.snippet_id.in_(wanted))))
+    async with async_session() as session:
+        idea_ids = [i for i in (await session.execute(query)).scalars().all()
+                    if await _is_member(session, project_id, i)]
+    moved: list[dict] = []
+    for idea_id in sorted(idea_ids):
+        if not await access.can_read_note(user_id, idea_id):
+            continue
+        async with async_session() as session:
+            idea = await session.get(FamilyIdea, idea_id)
+            said = shapes_say(await _shape_evidence(session, project_id, idea_id))
+            row = await _row(session, project_id, idea_id)
+            current = _row_state(row) if row is not None else None
+            source = await _latest_source(session, project_id, idea_id)
+        if said is not None:
+            if current and current["status"] == said["outcome"] \
+                    and current["canon_version"] == idea.canon_version:
+                continue
+            result = await assess(
+                user_id, project_id, idea_id, outcome=said["outcome"], reason=said["reason"],
+                evidence=said["evidence"], decided_via="system", source=SHAPE_SOURCE,
+                shapes=said["shapes"],
+            )
+            if result["changed"]:
+                moved.append({"idea_id": idea_id, "status": said["outcome"],
+                              "decision_id": result["decision"]["id"]})
+        elif current and current["status"] in ("adopted", "variant") and source == SHAPE_SOURCE:
+            decision = await _withdraw_shape_answer(user_id, project_id, idea_id)
+            moved.append({"idea_id": idea_id, "status": "unassessed", "decision_id": decision.id})
+    return moved
+
+
 async def assessment_precedents(user_id: int, project_id: int, idea_id: int,
                                 limit: int = 5) -> list[dict]:
     """What an assessment should be consistent with: the latest answer to
@@ -388,6 +619,20 @@ async def adoption_matrix(
                 select(Note).where(Note.id.in_(task_ids), Note.deleted_at.is_(None))
             )).scalars().all()
         } if task_ids else {}
+        # The code that answers each cell (step 5): live shapes classified
+        # against one of the idea's references.
+        evidence: dict[tuple[int, int], list[dict]] = {}
+        for iid, shape in (await session.execute(
+            select(FamilyIdeaReference.idea_id, CodeShape)
+            .join(CodeShape, CodeShape.snippet_id == FamilyIdeaReference.snippet_id)
+            .where(FamilyIdeaReference.idea_id.in_(idea_ids),
+                   CodeShape.project_id.in_(list(project_ids)),
+                   CodeShape.vanished_at.is_(None),
+                   CodeShape.status.in_(_SHAPE_ADOPTS + ("variant",)))
+            .order_by(CodeShape.path, CodeShape.symbol)
+        )).all() if idea_ids and project_ids else []:
+            evidence.setdefault((shape.project_id, iid), []).append(
+                {"path": shape.path, "symbol": shape.symbol, "status": shape.status})
 
     readable = [(pid, title) for pid, title in projects
                 if await access.can_read_project(user_id, pid)]
@@ -415,6 +660,7 @@ async def adoption_matrix(
                 "needs_recheck": bool(row) and needs_recheck(
                     row.status, row.canon_version, idea.status, idea.canon_version),
                 "owed_task": None,
+                "shapes": evidence.get((pid, iid), []),
             }
             task = tasks.get(row.owed_task_id) if row and row.owed_task_id else None
             if task is not None:
@@ -643,13 +889,17 @@ def _answer(row: FamilyAdoption, *, status: str, reason: str, version: int,
 async def assess(
     user_id: int, project_id: int, idea_id: int, *, outcome: str, reason: str,
     evidence: list[str] | None = None, precedent_ids: list[int] | None = None,
-    decided_via: str = "agent",
+    decided_via: str = "agent", source: str = "", shapes: list[dict] | None = None,
 ) -> dict:
     """Record one project's answer to one canon idea.
 
     Raises ValueError before writing anything when the answer is malformed
-    (assessment_problems), the caller cannot write the project, or the idea
-    is not canon or does not reach the project.
+    (assessment_problems), the caller cannot write the project, the idea
+    is not canon or does not reach the project, or the project's shape
+    ledger already answers the idea otherwise (the two never disagree).
+
+    ``source``/``shapes`` are the shape ledger's own door (sync_from_shapes):
+    the decision's evidence then names the shapes it was read from.
 
     The same answer given again — same outcome, reason and canon version —
     records nothing (`changed: False`); it still makes sure an owed answer
@@ -673,6 +923,11 @@ async def assess(
         if idea is None or idea.status != "canon":
             raise ValueError(f"#{idea_id} is not family canon — only canon is assessed")
         row = await _row_for_write(session, project_id, idea_id)
+        if source != SHAPE_SOURCE:
+            refused = _contradiction(
+                outcome, shapes_say(await _shape_evidence(session, project_id, idea_id)), idea_id)
+            if refused:
+                raise ValueError(refused)
         before = _row_state(row)
         after = {"status": outcome, "reason": reason.strip(), "canon_version": idea.canon_version}
         changed = before != after
@@ -683,8 +938,9 @@ async def assess(
             decision = family_svc._log(
                 session, idea_id=idea_id, project_id=project_id, action="assess",
                 reason=reason, before=before, after=_row_state(row),
-                evidence={"evidence": evidence}, precedent_ids=precedent_list,
-                decided_via=decided_via, user_id=user_id,
+                evidence=({"evidence": evidence, "source": source, "shapes": shapes or []}
+                          if source else {"evidence": evidence}),
+                precedent_ids=precedent_list, decided_via=decided_via, user_id=user_id,
             )
         await session.commit()
         if decision is not None:
@@ -722,6 +978,11 @@ async def undo_assessment(user_id: int, target: FamilyDecision, *, reason: str,
             raise ValueError(f"decision {target.id} cannot be undone: the answer it changed is gone")
         before = _row_state(row)
         prior = target.before or {"status": "unassessed", "reason": "", "canon_version": None}
+        refused = _contradiction(
+            prior["status"], shapes_say(await _shape_evidence(session, target.project_id, target.idea_id)),
+            target.idea_id)
+        if refused:
+            raise ValueError(f"decision {target.id} cannot be undone: {refused}")
         row.status = prior["status"]
         row.reason = (prior.get("reason") or "").strip() or None
         row.canon_version = prior.get("canon_version")
@@ -821,7 +1082,9 @@ async def resolve_conflict(
     the grounds checked above it); the canon side is answered `adopted`, and
     the other side `owed` (with a task filed) or, in a split, `adopted` under
     its condition. Each answer is its own `assess` decision, naming the
-    revision as its precedent.
+    revision as its precedent. Last, either side's shapes classified as a
+    variant of the idea are re-judged to agree with its new answer
+    (`shapes_rejudged`), so the shape ledger and this one do not disagree.
     """
     from scribe.services import notes as notes_svc
 
@@ -915,6 +1178,9 @@ async def resolve_conflict(
             row_ids[pid] = row.id
         await session.commit()
         await session.refresh(revision)
+    # The shapes follow the resolution, so the two ledgers still agree.
+    rejudged = {pid: await _shapes_follow(user_id, pid, idea_id, outcome, why)
+                for pid, outcome, why in answers}
     task = await _sync_owed_task(user_id, row_ids[other_project_id], reason=answers[1][2])
     await _sync_owed_task(user_id, row_ids[canon_project_id], reason=answers[0][2])
     return {
@@ -923,4 +1189,28 @@ async def resolve_conflict(
         "folded_as": g["fold_as"],
         "adoptions": await list_adoptions(user_id, idea_id=idea_id),
         "owed_task": task,
+        "shapes_rejudged": {str(pid): n for pid, n in rejudged.items() if n},
     }
+
+
+async def _shapes_follow(user_id: int, project_id: int, idea_id: int, outcome: str,
+                         why: str) -> int:
+    """Re-judge a project's variant shapes of the idea to agree with the
+    answer a conflict resolution just gave it: `adopted` makes them instances
+    of the reference they departed from (in a split, their condition is now
+    canon); `owed` returns them to the shape todo, to be rebuilt to the
+    canon the owed task names. Returns how many rows were re-judged."""
+    from scribe.services import shape_ledger
+
+    async with async_session() as session:
+        rows = [r for r in await _shape_evidence(session, project_id, idea_id)
+                if r.status == "variant"]
+    if not rows:
+        return 0
+    status = "instance" if outcome == "adopted" else "unclassified"
+    result = await shape_ledger.classify_shapes(user_id, project_id, [
+        {"path": r.path, "symbol": r.symbol, "kind": r.kind, "status": status,
+         "snippet_id": r.snippet_id, "reason": why}
+        for r in rows
+    ], via="agent")
+    return int(result.get("classified") or 0)