from datetime import datetime, timezone from pgvector.sqlalchemy import Vector from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, Text from sqlalchemy.orm import Mapped, mapped_column from scribe.models import Base # bge-small-en-v1.5 produces 384-dim unit-normalized vectors. The column is a # native pgvector `vector(384)` (see migration 0067) so similarity search runs # as an indexed `ORDER BY embedding <=> :q LIMIT k` in Postgres rather than a # full-table Python cosine scan. EMBEDDING_DIM = 384 class NoteEmbedding(Base): """One embedding vector per CHUNK of a note (#280, migration 0077). The model reads at most 512 tokens, so a single whole-document vector permanently lost everything past ~400 words. A note now stores one row per chunk of `embeddings.chunk_document`, and a query matches the note if it matches ANY chunk — retrieval collapses rows to best-chunk-per-note. A short note has exactly one row (chunk_index 0) whose text is the historical `title\\nbody` shape. """ __tablename__ = "note_embeddings" note_id: Mapped[int] = mapped_column( Integer, ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True, ) chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True) user_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False) # Exactly what this vector encodes — inspectable when a ranking surprises, # and the hook for surfacing WHICH section matched, later. chunk_text: Mapped[str] = mapped_column(Text, nullable=False) # embeddings.CHUNKER_VERSION at write time. The startup backfill re-embeds # any note whose rows carry a stale version — shape changes become a # version bump instead of a table wipe. chunker_version: Mapped[int] = mapped_column(Integer, nullable=False) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), ) class RuleEmbedding(Base): """One embedding vector per CHUNK of a rule (milestone 307, note 3026). A SIBLING of NoteEmbedding rather than a generalisation of it, decided deliberately. The reasoning below turned out to be the only written statement of a rule Scribe applies to every record type, so it is now also NOTE 3163 — "When a record type earns its own table" — with the worked cases and the cost of leaving `notes`. Read that before splitting a record type off; this stays here because it is where the decision was made. - The embedding ROW could have been made polymorphic. The SEARCH could not. `semantic_search_notes` is a long function of Note-specific scoping — the visibility clause, the supersession penalty, note_type/task_kind and system filters — and a rule shares none of it. Rules scope by rulebook ownership and project applicability instead. - Generalising the row while still needing two searches is the worst of both: a polymorphic key with referential integrity to neither table, on the path every session start runs, to share four columns. - What is genuinely common is BEHAVIOUR, not storage — get_embedding, chunk_document, embedding_text and CHUNKER_VERSION are already free functions and are reused as-is. Sharing those is the DRY win; sharing the table would have been the DRY costume. No `user_id`: NoteEmbedding carries one and its own search deliberately ignores it (scoping on the note instead, or shared records become unreachable). Rather than repeat a column that exists to be ignored, a rule's reach is resolved by joining the rule. """ __tablename__ = "rule_embeddings" rule_id: Mapped[int] = mapped_column( BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True, ) chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True) embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False) # Exactly what this vector encodes — inspectable when a ranking surprises. # For a rule this is the trigger-first document, NOT the rule's `why`: # `why` is dated incident narrative and would drag every rule toward one # centroid (measured in note 2485). chunk_text: Mapped[str] = mapped_column(Text, nullable=False) chunker_version: Mapped[int] = mapped_column(Integer, nullable=False) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), )