feat(snippets): add notes.data JSONB — the indexed mirror of snippet fields

Milestone #232 step 1 (task #2081). Takes the enabler first rather than the
write-path trigger: reverse lookup, drift checks and the duplicate finder all
need to QUERY structured fields, and building them on body-regex first means
writing them twice.

#227 deferred this bag "unless body-convention ergonomics prove insufficient" —
answering "which snippets live in this file?" by scanning every snippet and
regexing its body is that condition being met.

Migration 0070 adds `notes.data` (nullable JSONB) + a GIN index. The body is
UNCHANGED and still what gets embedded and read by humans; `data` mirrors the
same facts in a shape Postgres can index. Code is deliberately not copied into
it — the body holds it, and duplicating a blob into the column we index around
would be waste.

- compose_data() builds the mirror, omitting empties so the column stays sparse
- snippet_fields() prefers `data`, falling back to parsing the body. Rows written
  before 0070 have no `data` and are never backfilled, so a hand-edited body
  stays authoritative for them with no conversion deadline
- create / update / merge all write body and mirror from the same merged field
  set, so the two can't drift; merge in particular has to grow the mirror with
  the survivor's location set or a merged snippet would be unfindable at the very
  call sites the merge just recorded

Named `data`, not `metadata`, because that collides with SQLAlchemy's declarative
Base.metadata — which is why the pre-0069 model had to map an awkward
`entity_metadata` attribute. Not a revival of the column 0069 dropped: different
name, different purpose, nothing reads the old shape.

Two test fakes needed an explicit `data = None`: snippet_fields prefers `data`
when truthy and an auto-MagicMock attribute is truthy, so every parsed field
would have come back a MagicMock. Checked every fake reaching snippet code this
time rather than waiting for CI (note 2109).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLwAaV4DQEmVyn496HnEvt

src/scribe/models/note.py

@@ -69,6 +69,13 @@ class Note(Base, TimestampMixin, SoftDeleteMixin):
     # notes keep the 'work' default and ignore it. Orthogonal to note_type
     # (which is the note/entity axis).
     task_kind: Mapped[str] = mapped_column(Text, default="work", server_default="work")
+    # Queryable structured fields for typed records — currently snippets, whose
+    # name/language/signature/locations live here so they can be INDEXED. The
+    # body keeps the same facts in readable markdown and remains what gets
+    # embedded; this is a mirror for querying, not the source of truth for
+    # display. NULL on every row written before migration 0070, so readers fall
+    # back to parsing the body (see services/snippets.snippet_fields).
+    data: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
 
     __table_args__ = (
         Index("ix_notes_tags", "tags", postgresql_using="gin"),
@@ -79,6 +86,9 @@ class Note(Base, TimestampMixin, SoftDeleteMixin):
         Index("ix_notes_milestone_id", "milestone_id"),
         Index("ix_notes_note_type", "note_type"),
         Index("ix_notes_arose_from_id", "arose_from_id"),
+        # Containment queries into `data` — e.g. which snippets name a given
+        # repo/path in their locations. See migration 0070.
+        Index("ix_notes_data_gin", "data", postgresql_using="gin"),
     )
 
     @property

src/scribe/services/notes.py

@@ -65,6 +65,7 @@ async def create_note(
     note_type: str = "note",
     task_kind: str = "work",
     arose_from_id: int | None = None,
+    data: dict | None = None,
 ) -> Note:
     # Validate status/priority here so the MCP create_task path (which passes
     # them straight through) can't persist an out-of-enum value that the REST
@@ -108,6 +109,7 @@ async def create_note(
             note_type=note_type,
             task_kind=task_kind,
             arose_from_id=arose_from_id,
+            data=data,
         )
         session.add(note)
         await session.commit()

src/scribe/services/snippets.py

@@ -250,11 +250,78 @@ def parse_snippet_fields(
     return fields
 
 
+# --- the queryable mirror (notes.data, migration 0070) -----------------------
+
+# Fields kept in `data`. Code is deliberately absent: the body already holds it,
+# and copying a blob into the column we index *around* would be pure weight.
+_DATA_FIELDS = ("name", "when_to_use", "signature", "language", "locations")
+
+
+def compose_data(
+    *,
+    name: str = "",
+    when_to_use: str = "",
+    signature: str = "",
+    language: str = "",
+    locations: list[dict] | None = None,
+) -> dict:
+    """Build the `notes.data` mirror of a snippet's structured fields.
+
+    Same facts as the body convention, in a shape Postgres can index — so
+    "which snippets live in this path?" is a containment query rather than a
+    regex over every body. Empty values are omitted so the column stays sparse
+    and containment matches don't trip over blanks.
+    """
+    out: dict = {}
+    for key, value in (
+        ("name", (name or "").strip()),
+        ("when_to_use", (when_to_use or "").strip()),
+        ("signature", (signature or "").strip()),
+        ("language", (language or "").strip().lower()),
+    ):
+        if value:
+            out[key] = value
+    locs = _normalize_locations(locations)
+    if locs:
+        out["locations"] = locs
+    return out
+
+
+def snippet_fields(note) -> dict:
+    """Structured fields for a snippet, preferring the indexed `data` column and
+    falling back to parsing the body.
+
+    Both paths must agree, because rows written before migration 0070 have no
+    `data` and are never backfilled — a hand-edited body is the authority for
+    those, and there is no deadline by which they must be converted. `code` only
+    ever comes from the body, since `data` doesn't carry it.
+    """
+    parsed = parse_snippet_fields(note.title, note.body, note.tags)
+    stored = getattr(note, "data", None)
+    if not stored:
+        return parsed
+    merged = dict(parsed)
+    for key in _DATA_FIELDS:
+        if stored.get(key):
+            merged[key] = stored[key]
+    # Keep the single-location back-compat mirror consistent with whichever
+    # location list won.
+    locs = merged.get("locations") or []
+    merged["repo"] = locs[0]["repo"] if locs else ""
+    merged["path"] = locs[0]["path"] if locs else ""
+    merged["symbol"] = locs[0]["symbol"] if locs else ""
+    return merged
+
+
 def snippet_to_dict(note) -> dict:
     """Note serialization plus a parsed ``snippet`` sub-object of structured
-    fields, so callers get both the raw record and the typed view."""
+    fields, so callers get both the raw record and the typed view.
+
+    The raw `data` column is intentionally NOT exposed: it mirrors what
+    ``snippet`` already reports, and shipping both would give API consumers two
+    sources of truth for the same facts."""
     data = note.to_dict()
-    data["snippet"] = parse_snippet_fields(note.title, note.body, note.tags)
+    data["snippet"] = snippet_fields(note)
     return data
 
 
@@ -278,17 +345,24 @@ async def create_snippet(
     """Create a snippet note (embedded on create for immediate recall). Returns
     the created Note. Pass ``locations`` for the multi-location case; the single
     ``repo``/``path``/``symbol`` are the one-location shorthand."""
+    if locations is None:
+        locations = [{"repo": repo, "path": path, "symbol": symbol}]
     note = await notes_svc.create_note(
         user_id,
         title=compose_title(name, when_to_use),
         body=compose_body(
             code=code, language=language, signature=signature,
-            when_to_use=when_to_use, repo=repo, path=path, symbol=symbol,
-            locations=locations,
+            when_to_use=when_to_use, locations=locations,
         ),
         note_type=SNIPPET_NOTE_TYPE,
         tags=compose_tags(language, tags),
         project_id=project_id,
+        # The indexed mirror of the same fields (0070). Written together with the
+        # body so the two can never describe different things.
+        data=compose_data(
+            name=name, when_to_use=when_to_use, signature=signature,
+            language=language, locations=locations,
+        ),
     )
     _embed_snippet(note)
     return note
@@ -379,7 +453,7 @@ async def update_snippet(
             f"for edit access, or record your own version"
         )
 
-    cur = parse_snippet_fields(note.title, note.body, note.tags)
+    cur = snippet_fields(note)
     overlay = {
         "name": name, "code": code, "language": language,
         "signature": signature, "when_to_use": when_to_use,
@@ -405,6 +479,13 @@ async def update_snippet(
             signature=merged["signature"], when_to_use=merged["when_to_use"],
             locations=merged_locations,
         ),
+        # Re-derived from the same merged field set as the body, so an edit can't
+        # leave the indexed mirror describing the previous version.
+        "data": compose_data(
+            name=merged["name"], when_to_use=merged["when_to_use"],
+            signature=merged["signature"], language=merged["language"],
+            locations=merged_locations,
+        ),
     }
     # Recompute tags: keep any non-language, non-marker tags the note already had
     # (or the caller's replacement set), then re-derive language + marker.
@@ -511,10 +592,8 @@ async def merge_snippets(user_id: int, target_id: int, source_ids: list[int]):
             continue
         sources.append(s)
 
-    tgt_fields = parse_snippet_fields(target.title, target.body, target.tags)
-    parsed_sources = [
-        (parse_snippet_fields(s.title, s.body, s.tags), s.tags) for s in sources
-    ]
+    tgt_fields = snippet_fields(target)
+    parsed_sources = [(snippet_fields(s), s.tags) for s in sources]
     locations, extra_tags = merge_snippet_fields(tgt_fields, target.tags, parsed_sources)
 
     # Owner-scoped write, authorised above — same reason as update_snippet.
@@ -526,6 +605,14 @@ async def merge_snippets(user_id: int, target_id: int, source_ids: list[int]):
             locations=locations,
         ),
         tags=compose_tags(tgt_fields["language"], extra_tags),
+        # The survivor's location set grew, so its mirror has to grow with it —
+        # otherwise a merged snippet would be unfindable at the very call sites
+        # the merge just recorded.
+        data=compose_data(
+            name=tgt_fields["name"], when_to_use=tgt_fields["when_to_use"],
+            signature=tgt_fields["signature"], language=tgt_fields["language"],
+            locations=locations,
+        ),
     )
     if updated is None:
         return None