From 19fdc9aa8972494a5b4057c6196007b83f4d372d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 19 Aug 2026 19:25:25 -0400 Subject: [PATCH] =?UTF-8?q?feat(ledger):=20code=5Fshapes=20=E2=80=94=20the?= =?UTF-8?q?=20shape=20ledger=20schema=20(#2787,=20milestone=20294=20step?= =?UTF-8?q?=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accounting half of the pattern system (governing note 2786): the snippet library records canon (small), this table accounts for EVERY extracted shape (total). Identity is (project, repo_key, path, symbol, kind) — kind included because one file can define '.foo' (css) and 'foo' (sym) as distinct shapes. Status vocabulary: canonical / instance / variant / exempt / unclassified, with unclassified as the default and THE todo state; classifications carry who judged (agent|audit|hook|mechanical|import), when, and the why for variants/exemptions. first/last-seen commits + vanished_at keep history instead of deleting it; a rename reads as vanish+new (accepted for v1). snippet_id is SET NULL on snippet deletion so accounting rows outlive their target and rejoin the todo via the step-2 sync, never dangle silently. Backups: v7 carries code_shapes (judgment data, worth moving) — full and per-user export sections, and a restore that keeps a judgment only when its snippet survives the id re-mapping, downgrading to unclassified otherwise. Co-Authored-By: Claude Fable 5 --- alembic/versions/0079_code_shapes_ledger.py | 66 ++++++++++++++ src/scribe/models/__init__.py | 1 + src/scribe/models/code_shape.py | 98 +++++++++++++++++++++ src/scribe/services/backup.py | 58 +++++++++++- tests/test_services_backup.py | 16 ++-- tests/test_shape_ledger.py | 52 +++++++++++ 6 files changed, 282 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/0079_code_shapes_ledger.py create mode 100644 src/scribe/models/code_shape.py create mode 100644 tests/test_shape_ledger.py diff --git a/alembic/versions/0079_code_shapes_ledger.py b/alembic/versions/0079_code_shapes_ledger.py new file mode 100644 index 0000000..e548bc2 --- /dev/null +++ b/alembic/versions/0079_code_shapes_ledger.py @@ -0,0 +1,66 @@ +"""The shape ledger: code_shapes (#2787, milestone 294) + +Revision ID: 0079 +Revises: 0078 +Create Date: 2026-08-19 + +The accounting half of the pattern system (governing note 2786): the snippet +library records canon (small); this table accounts for EVERY shape the +coverage extractor finds in a bound repo (total). Rows arrive `unclassified` +from the coverage sync (step 2) and gain judgments — canonical / instance / +variant / exempt — from audits, hooks, and the mechanical proposer. +Unclassified IS the todo list. +""" +import sqlalchemy as sa +from alembic import op + +revision = "0079" +down_revision = "0078" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "code_shapes", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "project_id", + sa.Integer(), + sa.ForeignKey("projects.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("repo_key", sa.Text(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("symbol", sa.Text(), nullable=False), + sa.Column("kind", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False, server_default="unclassified"), + sa.Column( + "snippet_id", + sa.BigInteger(), + sa.ForeignKey("notes.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("classified_by", sa.Text(), nullable=True), + sa.Column("classified_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("first_seen_commit", sa.Text(), nullable=False, server_default=""), + sa.Column("last_seen_commit", sa.Text(), nullable=False, server_default=""), + sa.Column("vanished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint( + "project_id", "repo_key", "path", "symbol", "kind", + name="uq_code_shapes_identity", + ), + ) + op.create_index( + "ix_code_shapes_project_status", "code_shapes", ["project_id", "status"] + ) + op.create_index("ix_code_shapes_snippet", "code_shapes", ["snippet_id"]) + + +def downgrade() -> None: + op.drop_index("ix_code_shapes_snippet", table_name="code_shapes") + op.drop_index("ix_code_shapes_project_status", table_name="code_shapes") + op.drop_table("code_shapes") diff --git a/src/scribe/models/__init__.py b/src/scribe/models/__init__.py index 8690c07..ae7309c 100644 --- a/src/scribe/models/__init__.py +++ b/src/scribe/models/__init__.py @@ -44,5 +44,6 @@ from scribe.models.rulebook import ( # noqa: E402, F401 ) from scribe.models.repo_binding import RepoBinding # noqa: E402, F401 from scribe.models.forge_connection import ForgeConnection # noqa: E402, F401 +from scribe.models.code_shape import CodeShape # noqa: E402, F401 from scribe.models.system import System, RecordSystem # noqa: E402, F401 from scribe.models.design_system import DesignSystem, DesignToken # noqa: E402, F401 diff --git a/src/scribe/models/code_shape.py b/src/scribe/models/code_shape.py new file mode 100644 index 0000000..7999618 --- /dev/null +++ b/src/scribe/models/code_shape.py @@ -0,0 +1,98 @@ +from datetime import datetime + +from sqlalchemy import ( + BigInteger, + DateTime, + ForeignKey, + Index, + Integer, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column + +from scribe.models import Base +from scribe.models.base import TimestampMixin + +# The classification vocabulary (note 2786). `unclassified` is the default and +# THE todo state; every other status is a judgment, stamped with who made it. +SHAPE_STATUSES = ("canonical", "instance", "variant", "exempt", "unclassified") +SHAPE_CLASSIFIERS = ("agent", "audit", "hook", "mechanical", "import") + + +class CodeShape(Base, TimestampMixin): + """One extracted code shape and its classification against canon (#2787). + + The accounting half of the pattern system (governing note 2786): the + snippet library records CANON (small); this ledger accounts for EVERY + shape the coverage extractor finds in a bound repo (total). A row's + status says how the shape relates to canon — it IS a snippet's reference + (`canonical`), conforms to one (`instance` — snippet_id may point at + another project's snippet, so family canon counts), departs deliberately + (`variant`, with the why in `reason`), was judged one-off (`exempt`, + a recorded judgment rather than silence), or awaits judgment + (`unclassified` — the todo). + + Identity is (project, repo_key, path, symbol, kind) — kind is part of it + because one file can define `.foo` (css) and `foo` (sym) as distinct + shapes. A rename therefore reads as vanish + new row: accepted for v1, + because chasing renames needs content identity the extractor doesn't + have. `vanished_at` keeps the history instead of deleting it. + + snippet_id is SET NULL on snippet deletion: the classification's target + is gone but the judgment happened; the sync pass (step 2) re-files such + rows as unclassified so they rejoin the todo instead of dangling. + """ + + __tablename__ = "code_shapes" + __table_args__ = ( + UniqueConstraint( + "project_id", "repo_key", "path", "symbol", "kind", + name="uq_code_shapes_identity", + ), + Index("ix_code_shapes_project_status", "project_id", "status"), + Index("ix_code_shapes_snippet", "snippet_id"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + project_id: Mapped[int] = mapped_column( + Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False + ) + repo_key: Mapped[str] = mapped_column(Text, nullable=False) + path: Mapped[str] = mapped_column(Text, nullable=False) + symbol: Mapped[str] = mapped_column(Text, nullable=False) + kind: Mapped[str] = mapped_column(Text, nullable=False) # "css" | "sym" + status: Mapped[str] = mapped_column(Text, nullable=False, default="unclassified") + snippet_id: Mapped[int | None] = mapped_column( + BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True + ) + reason: Mapped[str | None] = mapped_column(Text, nullable=True) + classified_by: Mapped[str | None] = mapped_column(Text, nullable=True) + classified_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + first_seen_commit: Mapped[str] = mapped_column(Text, default="") + last_seen_commit: Mapped[str] = mapped_column(Text, default="") + vanished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + + def to_dict(self) -> dict: + return { + "id": self.id, + "project_id": self.project_id, + "repo_key": self.repo_key, + "path": self.path, + "symbol": self.symbol, + "kind": self.kind, + "status": self.status, + "snippet_id": self.snippet_id, + "reason": self.reason, + "classified_by": self.classified_by, + "classified_at": self.classified_at.isoformat() if self.classified_at else None, + "first_seen_commit": self.first_seen_commit, + "last_seen_commit": self.last_seen_commit, + "vanished_at": self.vanished_at.isoformat() if self.vanished_at else None, + "created_at": self.created_at.isoformat(), + "updated_at": self.updated_at.isoformat(), + } diff --git a/src/scribe/services/backup.py b/src/scribe/services/backup.py index 109796f..97edde3 100644 --- a/src/scribe/services/backup.py +++ b/src/scribe/services/backup.py @@ -11,6 +11,7 @@ from scribe.models.note_supersession import NoteSupersession from scribe.models.note_version import NoteVersion from scribe.models.design_system import DesignSystem, DesignToken from scribe.models.note_usage import NoteUsageEvent +from scribe.models.code_shape import CodeShape from scribe.models.project import Project from scribe.models.repo_binding import RepoBinding from scribe.models.rulebook import ( @@ -36,8 +37,11 @@ logger = logging.getLogger(__name__) # v6 (2026-08) added note_supersessions — and the guard did stop the seventh: # the table shipped without a backup section and the coverage test failed the # build, which is the whole reason that list was written. +# v7 (2026-08) added code_shapes — the shape ledger (#2787). Classifications +# are judgment data worth carrying; a restore keeps a judgment only when its +# snippet target survives the id re-mapping, else the row rejoins the todo. # Bump when the serialized schema changes. -BACKUP_VERSION = 6 +BACKUP_VERSION = 7 # Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED # below, these two lists must together account for the entire schema — which is @@ -55,6 +59,8 @@ _BACKED_UP = [ # v5 (2026-08): the five-year gap this list was written to stop. "systems", "record_systems", "design_systems", "design_tokens", "note_usage_events", "repo_bindings", "note_supersessions", + # v7 (2026-08): the shape ledger (#2787). + "code_shapes", ] # Tables intentionally NOT in the backup, surfaced in the payload so the gap is @@ -166,6 +172,10 @@ def _usage_event_rows(rows) -> list[dict]: ] +def _code_shape_rows(rows) -> list[dict]: + return [r.to_dict() for r in rows] + + def _repo_binding_rows(rows) -> list[dict]: return [ {"user_id": r.user_id, "project_id": r.project_id, "repo_key": r.repo_key} @@ -205,6 +215,7 @@ async def export_full_backup() -> dict: design_tokens = (await session.execute(select(DesignToken))).scalars().all() usage_events = (await session.execute(select(NoteUsageEvent))).scalars().all() repo_bindings = (await session.execute(select(RepoBinding))).scalars().all() + code_shapes = (await session.execute(select(CodeShape))).scalars().all() rulebooks = (await session.execute(select(Rulebook))).scalars().all() topics = (await session.execute(select(RulebookTopic))).scalars().all() rules = (await session.execute(select(Rule))).scalars().all() @@ -379,6 +390,7 @@ async def export_full_backup() -> dict: "note_usage_events": _usage_event_rows(usage_events), "repo_bindings": _repo_binding_rows(repo_bindings), "note_supersessions": _note_supersession_rows(supersessions), + "code_shapes": _code_shape_rows(code_shapes), } @@ -446,6 +458,11 @@ async def export_user_backup(user_id: int) -> dict: repo_bindings = (await session.execute( select(RepoBinding).where(RepoBinding.user_id == user_id) )).scalars().all() + # The ledger has no user_id of its own — rows belong to the project + # they account for, so a user's export carries their projects' rows. + code_shapes = (await session.execute( + select(CodeShape).where(CodeShape.project_id.in_(project_ids)) + )).scalars().all() if project_ids else [] rulebooks = (await session.execute( select(Rulebook).where(Rulebook.owner_user_id == user_id) )).scalars().all() @@ -634,6 +651,7 @@ async def export_user_backup(user_id: int) -> dict: "note_usage_events": _usage_event_rows(usage_events), "repo_bindings": _repo_binding_rows(repo_bindings), "note_supersessions": _note_supersession_rows(supersessions), + "code_shapes": _code_shape_rows(code_shapes), } @@ -737,7 +755,7 @@ async def _restore_v2(data: dict) -> dict: "topic_suppressions": 0, "systems": 0, "record_systems": 0, "design_systems": 0, "design_tokens": 0, "note_usage_events": 0, "repo_bindings": 0, - "note_supersessions": 0, + "note_supersessions": 0, "code_shapes": 0, } async with async_session() as session: @@ -1114,6 +1132,42 @@ async def _restore_v2(data: dict) -> dict: )) stats["repo_bindings"] += 1 + # 21. Code shapes (v7, #2787) — the ledger's classifications are + # judgments worth carrying. A judgment whose snippet target didn't + # survive the re-mapping (canonical/instance/variant with a gone + # snippet) is downgraded to unclassified so it rejoins the todo + # honestly instead of dangling; exempt needs no target and keeps. + for cs_data in data.get("code_shapes", []): + mapped_pid = project_id_map.get(cs_data.get("project_id", 0)) + if mapped_pid is None: + continue + status = cs_data.get("status", "unclassified") + mapped_sid = note_id_map.get(cs_data.get("snippet_id") or 0) + classified_by = cs_data.get("classified_by") + classified_at = cs_data.get("classified_at") + if status in ("canonical", "instance", "variant") and mapped_sid is None: + status = "unclassified" + classified_by = None + classified_at = None + session.add(CodeShape( + project_id=mapped_pid, + repo_key=cs_data.get("repo_key", ""), + path=cs_data.get("path", ""), + symbol=cs_data.get("symbol", ""), + kind=cs_data.get("kind", "sym"), + status=status, + snippet_id=mapped_sid, + reason=cs_data.get("reason"), + classified_by=classified_by, + classified_at=_dt(classified_at) if classified_at else None, + first_seen_commit=cs_data.get("first_seen_commit", ""), + last_seen_commit=cs_data.get("last_seen_commit", ""), + vanished_at=_dt(cs_data["vanished_at"]) if cs_data.get("vanished_at") else None, + created_at=_dt(cs_data.get("created_at")), + updated_at=_dt(cs_data.get("updated_at")), + )) + stats["code_shapes"] += 1 + await session.commit() logger.info("Restored v2/v3 backup: %s", stats) diff --git a/tests/test_services_backup.py b/tests/test_services_backup.py index 914760b..6978d87 100644 --- a/tests/test_services_backup.py +++ b/tests/test_services_backup.py @@ -13,17 +13,19 @@ import pytest from scribe.services import backup -def test_backup_version_is_v6(): - """v6 added note_supersessions (#278). The bump is the point of the test — +def test_backup_version_is_v7(): + """v7 added code_shapes (#2787). The bump is the point of the test — a payload section added without moving the version produces backups that are structurally different and indistinguishable by inspection.""" - assert backup.BACKUP_VERSION == 6 + assert backup.BACKUP_VERSION == 7 def test_not_included_lists_the_known_gaps(): # The deferred tables must be surfaced explicitly, not silently dropped. + # forge_connections is excluded as CREDENTIALS (api_keys reasoning): a + # backup that carries forge tokens is a token-exfiltration file (#2778). for table in ("groups", "project_shares", "note_shares", "api_keys", - "note_embeddings", "retrieval_logs"): + "note_embeddings", "retrieval_logs", "forge_connections"): assert table in backup._NOT_INCLUDED @@ -105,14 +107,14 @@ async def test_export_full_backup_contains_every_declared_section(): assert out["version"] == backup.BACKUP_VERSION assert out["scope"] == "full" assert "api_keys" in out["_not_included"] - # The sections v2 silently dropped, the six v5 added, and v6's - # note_supersessions (all empty here). + # The sections v2 silently dropped, the six v5 added, v6's + # note_supersessions, and v7's code_shapes (all empty here). for key in ("rulebooks", "rulebook_topics", "rules", "rulebook_subscriptions", "rule_suppressions", "topic_suppressions", "systems", "record_systems", "design_systems", "design_tokens", "note_usage_events", "repo_bindings", - "note_supersessions"): + "note_supersessions", "code_shapes"): assert key in out, f"missing export section: {key}" assert out[key] == [] diff --git a/tests/test_shape_ledger.py b/tests/test_shape_ledger.py new file mode 100644 index 0000000..07ecbd4 --- /dev/null +++ b/tests/test_shape_ledger.py @@ -0,0 +1,52 @@ +"""The shape ledger's schema contract (#2787, milestone 294, note 2786). + +Step 1 pins the model: identity, the classification vocabulary, and the +token-free serialisation. The sync pass (step 2) and the classification +surface (step 3) grow their tests here; DB-backed behavior lands in the +integration lane once there is behavior to exercise. +""" +from scribe.models import Base +from scribe.models.code_shape import SHAPE_CLASSIFIERS, SHAPE_STATUSES, CodeShape + + +def test_identity_is_project_repo_path_symbol_kind(): + """Kind is part of identity on purpose: one file can define `.foo` (css) + and `foo` (sym) as distinct shapes — the extractor emits both.""" + table = Base.metadata.tables["code_shapes"] + unique = next( + c for c in table.constraints + if getattr(c, "name", "") == "uq_code_shapes_identity" + ) + assert [c.name for c in unique.columns] == [ + "project_id", "repo_key", "path", "symbol", "kind", + ] + + +def test_the_todo_state_is_the_default(): + """A shape nobody has judged yet must read `unclassified` — the ledger's + todo list — never silently look classified.""" + assert CodeShape.__table__.c.status.default.arg == "unclassified" + assert "unclassified" in SHAPE_STATUSES + assert set(SHAPE_STATUSES) == { + "canonical", "instance", "variant", "exempt", "unclassified", + } + assert set(SHAPE_CLASSIFIERS) == { + "agent", "audit", "hook", "mechanical", "import", + } + + +def test_snippet_reference_survives_snippet_deletion_as_null(): + """SET NULL, not CASCADE: a deleted snippet must not silently erase the + accounting rows that pointed at it — the sync pass re-files them as + unclassified so they rejoin the todo.""" + fk = next(iter(CodeShape.__table__.c.snippet_id.foreign_keys)) + assert fk.ondelete == "SET NULL" + assert fk.column.table.name == "notes" + + +def test_status_queries_have_an_index(): + """list_shapes(status=unclassified) is THE todo query (step 3) — it must + not degrade into a project-wide scan as ledgers reach thousands of rows.""" + names = {ix.name for ix in CodeShape.__table__.indexes} + assert "ix_code_shapes_project_status" in names + assert "ix_code_shapes_snippet" in names