rename(547): the server is grimoire — package, image, DB defaults, backups
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 56s
CI & Build / integration (push) Successful in 1m4s
CI & Build / Python tests (push) Successful in 1m55s
CI & Build / Build & push image (push) Successful in 44s

Step 2 of the Scribe -> Grimoire rename.

- src/scribe -> src/grimoire, and every import, patch target, module path,
  hypercorn entry point and `python -m grimoire.restore` with it; pyproject,
  uv.lock and the frontend package are named grimoire.
- Image git.fabledsword.com/bvandeusen/fabledgrimoire; CI's Postgres service
  runs as grimoire / grimoire_test.
- Compose files default the DB user, database and (prod) volume to grimoire,
  each overridable from .env, so an install created as Scribe keeps its data
  by setting POSTGRES_USER, POSTGRES_DB and PGDATA_VOLUME.
- Backups are written as grimoire-<ts>.tar with format grimoire-backup-tar.
  Archives named scribe-* with the scribe-backup-tar manifest still restore
  and are still counted at the destination (folder and S3), with tests.
- Exports, the MCP auth realm, the ASGI scope key and the persistent logger
  carry the new name.

Kept on purpose: the secret-sealing key label stays "scribe.secrets.v1",
because every stored secret was derived under it.

Plugin identity (plugin name, hooks, env vars, .scribe marker, skills) is
step 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-22 10:50:45 -04:00
co-authored by Claude Opus 5.5
parent e99f1169f7
commit 0f5ef0be63
447 changed files with 3611 additions and 3558 deletions
+96
View File
@@ -0,0 +1,96 @@
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from grimoire.config import Config
# Named rather than inlined so the deadline below is READABLE. SQLAlchemy
# captures `connect_args` in a closure and merges it at connect time, so an
# inline dict cannot be recovered from the engine — and a guard that cannot
# read the value it guards is a guard that passes forever.
_CONNECT_ARGS: dict = {
# A DEADLINE ON ESTABLISHING A CONNECTION.
#
# asyncpg's `timeout` bounds the CONNECT — the TCP handshake plus session
# setup — and nothing else. Against a host whose storage has wedged, that
# handshake does not fail, it waits, and without this the wait is
# asyncpg's own 60-second default: the entire lifespan budget spent before
# a single query is even sent. Ten seconds is far longer than a healthy
# local connect (single-digit milliseconds) and short enough to leave room
# to fail usefully rather than be killed.
#
# WHAT THIS DOES NOT COVER, said plainly so the next reader doesn't assume
# it does: a query on an already-open connection, which includes
# `pool_pre_ping`'s liveness check. Bounding those is `command_timeout`,
# and that is deliberately NOT set here — it would apply to every
# statement, and this app legitimately runs long ones (the embedding
# backfills, VACUUM ANALYZE). A blanket statement deadline would trade
# this failure mode for a worse one. Callers that must not hang — the
# lifespan hook above all — bound their own await instead; see
# `_STARTUP_READ_TIMEOUT` in services/db_maintenance_scheduler.py.
"timeout": 10,
}
engine = create_async_engine(
Config.DATABASE_URL,
echo=False,
pool_pre_ping=True,
pool_recycle=1800,
connect_args=_CONNECT_ARGS,
)
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
class Base(DeclarativeBase):
pass
from grimoire.models.base import CreatedAtMixin, TimestampMixin # noqa: E402, F401
from grimoire.models.note import Note, TaskPriority, TaskStatus # noqa: E402, F401
from grimoire.models.setting import Setting # noqa: E402, F401
from grimoire.models.user import User # noqa: E402, F401
from grimoire.models.app_log import AppLog # noqa: E402, F401
from grimoire.models.password_reset import PasswordResetToken # noqa: E402, F401
from grimoire.models.invitation import InvitationToken # noqa: E402, F401
from grimoire.models.embedding import ( # noqa: E402, F401
MilestoneEmbedding, NoteEmbedding, RuleEmbedding, SystemEmbedding,
)
from grimoire.models.retrieval_log import RetrievalLog # noqa: E402, F401
from grimoire.models.retrieval_tuning import RetrievalTuningEvent # noqa: E402, F401
from grimoire.models.note_usage import NoteUsageEvent # noqa: E402, F401
from grimoire.models.rule_usage import RuleUsageEvent # noqa: E402, F401
from grimoire.models.system_usage import SystemUsageEvent # noqa: E402, F401
from grimoire.models.retrieval_judgment import RetrievalJudgment # noqa: E402, F401
from grimoire.models.moment_mapping import MomentMapping # noqa: E402, F401
from grimoire.models.project import Project # noqa: E402, F401
from grimoire.models.milestone import Milestone # noqa: E402, F401
from grimoire.models.task_log import TaskLog # noqa: E402, F401
from grimoire.models.note_draft import NoteDraft # noqa: E402, F401
from grimoire.models.note_version import NoteVersion # noqa: E402, F401
from grimoire.models.rule_version import RuleVersion # noqa: E402, F401
from grimoire.models.note_supersession import NoteSupersession # noqa: E402, F401
from grimoire.models.group import Group, GroupMembership # noqa: E402, F401
from grimoire.models.share import NoteShare, ProjectShare # noqa: E402, F401
from grimoire.models.notification import Notification # noqa: E402, F401
from grimoire.models.api_key import ApiKey # noqa: E402, F401
# Imported before rulebook: rule_systems foreign-keys canonical_systems.
from grimoire.models.canonical_system import CanonicalSystem # noqa: E402, F401
from grimoire.models.rulebook import ( # noqa: E402, F401
Rulebook, RulebookTopic, Rule, RuleRelation, rule_moments, rule_systems,
)
# After notes and rules: it foreign-keys both.
from grimoire.models.lesson_rule_link import LessonNoRule, LessonRuleLink # noqa: E402, F401
# After rules: it foreign-keys them.
from grimoire.models.rule_moment_judgment import RuleMomentJudgment # noqa: E402, F401
from grimoire.models.repo_binding import RepoBinding # noqa: E402, F401
from grimoire.models.forge_connection import ForgeConnection # noqa: E402, F401
from grimoire.models.code_shape import CodeShape, CodeShapeConsumer, CodeShapeEvent, CodeShapeUse # noqa: E402, F401
from grimoire.models.system import System, RecordSystem # noqa: E402, F401
from grimoire.models.design_system import DesignSystem, DesignToken # noqa: E402, F401
# After notes, projects and rulebook topics: family canon foreign-keys all
# three.
from grimoire.models.family import ( # noqa: E402, F401
FamilyAdoption, FamilyDecision, FamilyIdea, FamilyIdeaPlatform,
FamilyIdeaReference, Platform, ProjectPlatform,
)
+42
View File
@@ -0,0 +1,42 @@
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class ApiKey(Base, CreatedAtMixin):
__tablename__ = "api_keys"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
name: Mapped[str] = mapped_column(Text, nullable=False)
key_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
key_prefix: Mapped[str] = mapped_column(Text, nullable=False)
scope: Mapped[str] = mapped_column(Text, nullable=False) # "read" or "write"
last_used_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
revoked_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
__table_args__ = (
Index("ix_api_keys_user_id", "user_id"),
Index("ix_api_keys_key_hash", "key_hash"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"name": self.name,
"key_prefix": self.key_prefix,
"scope": self.scope,
"last_used_at": iso(self.last_used_at),
"created_at": iso(self.created_at),
"revoked_at": iso(self.revoked_at),
}
+52
View File
@@ -0,0 +1,52 @@
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import iso
class AppLog(Base):
__tablename__ = "app_logs"
id: Mapped[int] = mapped_column(primary_key=True)
category: Mapped[str] = mapped_column(Text, nullable=False)
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
username: Mapped[str | None] = mapped_column(Text, nullable=True)
action: Mapped[str | None] = mapped_column(Text, nullable=True)
endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
method: Mapped[str | None] = mapped_column(Text, nullable=True)
status_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
duration_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
ip_address: Mapped[str | None] = mapped_column(Text, nullable=True)
details: Mapped[str | None] = mapped_column(Text, nullable=True)
# Declared here rather than via CreatedAtMixin on purpose: the composite
# index below orders on `created_at.desc()`, which needs the column object
# in this class body — a mixin's column is not in scope there.
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
__table_args__ = (
Index("ix_app_logs_category", "category"),
Index("ix_app_logs_user_id", "user_id"),
Index("ix_app_logs_created_at", "created_at"),
Index("ix_app_logs_category_created_at", "category", created_at.desc()),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"category": self.category,
"user_id": self.user_id,
"username": self.username,
"action": self.action,
"endpoint": self.endpoint,
"method": self.method,
"status_code": self.status_code,
"duration_ms": self.duration_ms,
"ip_address": self.ip_address,
"details": self.details,
"created_at": iso(self.created_at),
}
+40
View File
@@ -0,0 +1,40 @@
from datetime import date, datetime, timezone
from sqlalchemy import DateTime, Text
from sqlalchemy.orm import Mapped, mapped_column
def iso(value: datetime | date | None) -> str | None:
"""ISO-8601 for a payload, None for an unset column.
Every model's to_dict serialises timestamps through this one helper so a
row read before flush (created_at still None) and a nullable column both
come out as null instead of raising on `.isoformat()`.
"""
return value.isoformat() if value else None
class SoftDeleteMixin:
"""Recoverable-delete columns. NULL deleted_at = live row. deleted_batch_id
groups rows soft-deleted in one operation so a cascade restores as a unit."""
deleted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True, default=None
)
deleted_batch_id: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
class TimestampMixin:
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
onupdate=lambda: datetime.now(timezone.utc),
)
class CreatedAtMixin:
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
+58
View File
@@ -0,0 +1,58 @@
from sqlalchemy import Index, Integer, Text, text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import SoftDeleteMixin, TimestampMixin, iso
class CanonicalSystem(Base, TimestampMixin, SoftDeleteMixin):
"""A GLOBAL area name — the shared vocabulary every project's Systems
can point at.
A `System` is per-project and NOT NULL on project_id, so nothing outside a
project can reference one: a rule that applies across projects has no way
to say "this is about CI" without chaining itself to one project's row.
This table is that join key. It carries no `user_id` on purpose — a shared
project must INHERIT the vocabulary rather than re-earn it, so the catalog
is global and the same word means the same thing in every install.
It is a convergence aid, never a gate: `systems.canonical_id` is nullable,
an unmapped System stays fully usable, and `record_systems` never sees this
table at all — the local name is a legitimate local label and is never
rewritten to match.
`slug` is the match key, not a display value. It folds the spelling
differences that produced four names for one area on the author's own
instance ("CI & Release" / "CI and Release" / "CI & release"): an exact slug
hit maps automatically, and anything short of that becomes a proposal for a
human to confirm. See services/canonical_systems.canonical_slug.
"""
__tablename__ = "canonical_systems"
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(Text, nullable=False)
# Normalized match key — unique among LIVE rows, so a soft-deleted entry
# doesn't block recreating the same area (the partial-unique convention
# rules/topics already use).
slug: Mapped[str] = mapped_column(Text, nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
order_index: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
__table_args__ = (
Index(
"uq_canonical_systems_slug", "slug",
unique=True, postgresql_where=text("deleted_at IS NULL"),
),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"name": self.name,
"slug": self.slug,
"description": self.description,
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+367
View File
@@ -0,0 +1,367 @@
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger,
DateTime,
Float,
ForeignKey,
Index,
Integer,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
# The classification vocabulary. `unclassified` is the default and
# THE todo state; every other status is a judgment, stamped with who made it —
# except `scoped`: the coverage sync's mechanical stamp on shapes that
# are one-offs BY CONSTRUCTION (a Vue component's scoped <style> rules and its
# <script setup> functions — unreachable from any other file). Scoped rows are
# accounted for without a human judging them, so `exempt` keeps meaning "a
# person looked"; the proposer, derive grouping and divergence still see them,
# and any judgment (instance/variant/exempt) overrides the stamp.
SHAPE_STATUSES = ("canonical", "instance", "variant", "exempt", "scoped", "unclassified")
SHAPE_CLASSIFIERS = ("agent", "audit", "hook", "mechanical", "import")
# The reason catalogue: an OPTIONAL code beside the prose reason on
# variant/exempt rows, so the ledger can be filtered and aggregated by kind
# of one-off. The prose remains the record; the code is the index.
REASON_CODES = (
"scoped-css", # a scoped rule styling one element (rows judged before `scoped`)
"one-off-handler", # a view/component handler or loader, one per surface
"test-helper", # a test module's stub, driver or fixture data
"convention-plumbing", # registration, wiring, app factory — one of each
"pure-helper", # a sync module-private helper with no session
"generated", # generated source (theme.css, protos, bundles)
"script", # a standalone dev/CI script
"typed-record", # a NamedTuple / dataclass / error class — one each
)
# How the mechanical proposer arrived at a proposal, strongest first.
# `derive` is the odd one out: not "this is an instance of #N" but "this
# shape repeats with NO canon — derive one first", so it carries a group key instead of a snippet.
PROPOSAL_BASES = ("symbol", "text", "reference", "signature", "semantic", "derive")
class CodeShape(Base, TimestampMixin):
"""One extracted code shape and its classification against canon.
The accounting half of the pattern system: the
snippet library records CANON (small); this ledger accounts for EVERY
shape the coverage extractor finds in a bound repo (total). A row's
status says how the shape relates to canon — it IS a snippet's reference
(`canonical`), conforms to one (`instance` — snippet_id may point at
another project's snippet, so family canon counts), departs deliberately
(`variant`, with the why in `reason`), was judged one-off (`exempt`,
a recorded judgment rather than silence), or awaits judgment
(`unclassified` — the todo).
Identity is (project, repo_key, path, symbol, kind) — kind is part of it
because one file can define `.foo` (css) and `foo` (sym) as distinct
shapes. A rename therefore reads as vanish + new row: accepted,
because chasing renames needs content identity the extractor doesn't
have. `vanished_at` keeps the history instead of deleting it.
snippet_id is SET NULL on snippet deletion: the classification's target
is gone but the judgment happened; the sync pass re-files such
rows as unclassified so they rejoin the todo instead of dangling.
`signature` / `body_sha` are the shape's content fingerprint —
its definition line and a whitespace/comment-insensitive hash of its
block — refreshed by every sync. They are what the mechanical proposer
matches on and what a later drift recheck compares against; the ledger
still never stores code bodies.
`classified_sha` remembers the fingerprint a judgment was made at;
when a later sync sees the body change under an instance/variant, the
row is flagged `recheck_at` (the judgment stands, it just asks to be
confirmed again) and a `drifted` event is written. `diverges_from`
is the button-B flag: a shape new since the previous refresh, in
a directory+kind where one canon dominates the judged siblings, that the
proposer did not match to that canon — "button B appeared where button
A is canon: divergence or variant? classify it." Both clear on judgment.
The proposal columns hold the proposer's standing suggestion for an
UNCLASSIFIED row: `proposed_snippet_id` + `proposal_basis` + score for
"looks like an instance of #N", or `proposal_basis="derive"` +
`proposal_group` for "repeats with no canon". `proposed_sha` is the
body_sha the row was last examined at, so a refresh re-examines only
what changed. A judgment clears the proposal — the machine proposes,
judgment classifies.
"""
__tablename__ = "code_shapes"
__table_args__ = (
UniqueConstraint(
"project_id", "repo_key", "path", "symbol", "kind",
name="uq_code_shapes_identity",
),
Index("ix_code_shapes_project_status", "project_id", "status"),
Index("ix_code_shapes_snippet", "snippet_id"),
Index("ix_code_shapes_proposed", "project_id", "proposed_snippet_id"),
Index("ix_code_shapes_diverges", "project_id", "diverges_from"),
)
id: Mapped[int] = mapped_column(primary_key=True)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False
)
repo_key: Mapped[str] = mapped_column(Text, nullable=False)
path: Mapped[str] = mapped_column(Text, nullable=False)
symbol: Mapped[str] = mapped_column(Text, nullable=False)
kind: Mapped[str] = mapped_column(Text, nullable=False) # "css" | "sym"
status: Mapped[str] = mapped_column(Text, nullable=False, default="unclassified")
snippet_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
reason_code: Mapped[str | None] = mapped_column(Text, nullable=True) # REASON_CODES
classified_by: Mapped[str | None] = mapped_column(Text, nullable=True)
classified_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
first_seen_commit: Mapped[str] = mapped_column(Text, default="")
last_seen_commit: Mapped[str] = mapped_column(Text, default="")
vanished_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
signature: Mapped[str] = mapped_column(Text, default="")
body_sha: Mapped[str] = mapped_column(Text, default="")
proposed_snippet_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
proposal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
proposal_score: Mapped[float | None] = mapped_column(Float, nullable=True)
proposal_group: Mapped[str | None] = mapped_column(Text, nullable=True)
proposed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
proposed_sha: Mapped[str] = mapped_column(Text, default="")
classified_sha: Mapped[str] = mapped_column(Text, default="")
recheck_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
diverges_from: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
@property
def proposal(self) -> dict | None:
"""The standing proposal as one object, or None when the proposer
has nothing to say about this row."""
if self.proposed_snippet_id is None and not self.proposal_group:
return None
out: dict = {"basis": self.proposal_basis, "score": self.proposal_score}
if self.proposed_snippet_id is not None:
out["snippet_id"] = self.proposed_snippet_id
if self.proposal_group:
out["group"] = self.proposal_group
return out
def to_dict(self) -> dict:
return {
"id": self.id,
"project_id": self.project_id,
"repo_key": self.repo_key,
"path": self.path,
"symbol": self.symbol,
"kind": self.kind,
"status": self.status,
"snippet_id": self.snippet_id,
"reason": self.reason,
"reason_code": self.reason_code,
"classified_by": self.classified_by,
"classified_at": iso(self.classified_at),
"first_seen_commit": self.first_seen_commit,
"last_seen_commit": self.last_seen_commit,
"vanished_at": iso(self.vanished_at),
"signature": self.signature,
"body_sha": self.body_sha,
"proposal": self.proposal,
"classified_sha": self.classified_sha,
"recheck_at": iso(self.recheck_at),
"diverges_from": self.diverges_from,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
def to_compact(self) -> dict:
"""The row as an audit reads it: identity, standing, the
definition line and the proposer's word — none of the bookkeeping
(commits, shas, timestamps). A 500-row page of these fits the tool
budget; a page of to_dict() does not."""
out = {
"path": self.path,
"symbol": self.symbol,
"kind": self.kind,
"status": self.status,
"signature": self.signature,
}
if self.snippet_id is not None:
out["snippet_id"] = self.snippet_id
if self.classified_by:
out["by"] = self.classified_by
if self.reason_code:
out["reason_code"] = self.reason_code
proposal = self.proposal
if proposal:
out["proposal"] = proposal
if self.diverges_from is not None:
out["diverges_from"] = self.diverges_from
if self.recheck_at is not None:
out["recheck"] = True
return out
# How a uses edge was established: who/what said "this shape calls
# that canon". `reference` is the proposer's mechanical by-name hit on the
# body (language-gated); `hook` is write-path evidence (pulled the
# snippet, then wrote code naming its symbol); agent/audit/import are
# judgments carried on classify_shapes(..., uses=[...]).
USE_BASES = ("reference", "hook", "agent", "audit", "import")
class CodeShapeUse(Base):
"""One consumption edge: shape → canonical snippet it calls/uses.
Conformance (CodeShape.status/snippet_id) answers "what shape is this";
this table answers "what does it use" — many per shape. A service function
that is an instance of the service-function convention AND a consumer of
hash_token has one snippet_id and one uses edge. Cascades with both ends:
a use of a deleted snippet is no longer a fact worth keeping.
"""
__tablename__ = "code_shape_uses"
__table_args__ = (
UniqueConstraint("shape_id", "snippet_id", name="uq_code_shape_uses_shape_snippet"),
Index("ix_code_shape_uses_snippet", "snippet_id"),
)
id: Mapped[int] = mapped_column(primary_key=True)
shape_id: Mapped[int] = mapped_column(
Integer, ForeignKey("code_shapes.id", ondelete="CASCADE"), nullable=False
)
snippet_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), nullable=False
)
basis: Mapped[str] = mapped_column(Text, nullable=False)
evidence: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
def to_dict(self) -> dict:
return {
"id": self.id,
"shape_id": self.shape_id,
"snippet_id": self.snippet_id,
"basis": self.basis,
"evidence": self.evidence,
"created_at": iso(self.created_at),
}
# How a consumer edge was established. `template` is the
# sync's mechanical read of a file's markup (class= / :class= / className=);
# the vocabulary is a list so a later basis (a stylesheet `@apply`, a script's
# classList) has a name without a schema change.
CONSUMER_BASES = ("template",)
class CodeShapeConsumer(Base):
"""One consumer edge: CSS shape → the file whose markup names its class
(CSS is watched by name, by recipe, by token and by WHAT USES IT). The
analogue of CodeShapeUse for styling: `uses`
says what a shape calls, this says who renders a class. Rows, not prose,
so "is this recipe shared or scoped?" is a count, not a guess.
Mechanical and fully recomputable: every coverage sync rebuilds a repo's
edges from its archive, so the table is not backed up (see
services/backup._NOT_INCLUDED). Cascades with the shape.
"""
__tablename__ = "code_shape_consumers"
__table_args__ = (
UniqueConstraint("shape_id", "path", name="uq_code_shape_consumers_shape_path"),
)
id: Mapped[int] = mapped_column(primary_key=True)
shape_id: Mapped[int] = mapped_column(
Integer, ForeignKey("code_shapes.id", ondelete="CASCADE"), nullable=False
)
path: Mapped[str] = mapped_column(Text, nullable=False)
count: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
basis: Mapped[str] = mapped_column(Text, nullable=False, default="template")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
def to_dict(self) -> dict:
return {
"id": self.id,
"shape_id": self.shape_id,
"path": self.path,
"count": self.count,
"basis": self.basis,
"created_at": iso(self.created_at),
}
# What a shape's history records. Not "appeared" — first_seen and
# created_at already say that on the row; history is for what CHANGED:
SHAPE_EVENTS = ("classified", "vanished", "reappeared", "drifted")
class CodeShapeEvent(Base):
"""One state change in a shape's life — the what-was-used-when record.
"We used #N here from <date>, #M replaced it at commit C, reason R" is a
question the ledger row alone cannot answer once it has moved on; this
table keeps each judgment (status, snippet, who, why, at which commit)
and each presence change (vanished / reappeared / drifted) as it
happened. Denormalised path/symbol/kind so a directory's history reads
without joining; `snippet_id` is deliberately FK-free — history outlives
the snippet it names, which is the point.
"""
__tablename__ = "code_shape_events"
__table_args__ = (
Index("ix_code_shape_events_shape", "shape_id", "at"),
Index("ix_code_shape_events_project_path", "project_id", "path"),
)
id: Mapped[int] = mapped_column(primary_key=True)
shape_id: Mapped[int] = mapped_column(
Integer, ForeignKey("code_shapes.id", ondelete="CASCADE"), nullable=False
)
project_id: Mapped[int] = mapped_column(Integer, nullable=False)
path: Mapped[str] = mapped_column(Text, nullable=False)
symbol: Mapped[str] = mapped_column(Text, nullable=False)
kind: Mapped[str] = mapped_column(Text, nullable=False)
event: Mapped[str] = mapped_column(Text, nullable=False)
status: Mapped[str | None] = mapped_column(Text, nullable=True)
snippet_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
classified_by: Mapped[str | None] = mapped_column(Text, nullable=True)
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
commit: Mapped[str] = mapped_column(Text, default="")
at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
def to_dict(self) -> dict:
return {
"id": self.id,
"shape_id": self.shape_id,
"project_id": self.project_id,
"path": self.path,
"symbol": self.symbol,
"kind": self.kind,
"event": self.event,
"status": self.status,
"snippet_id": self.snippet_id,
"classified_by": self.classified_by,
"reason": self.reason,
"commit": self.commit,
"at": iso(self.at),
}
+158
View File
@@ -0,0 +1,158 @@
"""Design systems — a stylesheet held as records, inherited family -> app.
A DesignSystem is a named set of design tokens with an OPTIONAL parent, and that
single self-FK is the whole model. A system with no parent is a family system; a
system WITH one holds only what it changes. "What does this app alter?" is
therefore `list its tokens` — nothing to compute, nothing to diff — which is why
inheritance won over a flat family-plus-loose-overrides shape.
Resolution walks the chain and lets the deepest system win by token name. That
is the CSS cascade rather than an analogy to it, which is why the storage model
and the stylesheet model come out the same shape.
`parent_id` also replaces two things the rulebook model needs to express the same
idea: an `always_on` flag (a family system is simply one with no parent) and a
subscription join table (a project points at ONE system, and the chain supplies
the rest). Less schema for more structure.
"""
from sqlalchemy import BigInteger, ForeignKey, Index, Integer, Text, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import SoftDeleteMixin, TimestampMixin, iso
class DesignSystem(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "design_systems"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
owner_user_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("users.id", ondelete="CASCADE"), index=True
)
title: Mapped[str] = mapped_column(Text)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
# The narrative a token table cannot hold: aesthetic, voice and tone, what
# is deliberately out of scope. Free-form markdown rather than a column per
# category — a schema with `voice`/`aesthetic`/`scope` columns would bake one
# rulebook's table of contents into every install.
guidance: Mapped[str | None] = mapped_column(Text, nullable=True)
# SET NULL, not CASCADE: deleting a family system must not delete every app
# system that inherited from it. Orphaning turns each child into a root that
# still holds its own overrides — recoverable. A cascade would destroy data
# the operator never asked to touch.
parent_id: Mapped[int | None] = mapped_column(
BigInteger,
ForeignKey("design_systems.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
def to_dict(self) -> dict:
return {
"id": self.id,
"owner_user_id": self.owner_user_id,
"title": self.title,
"description": self.description or "",
"guidance": self.guidance or "",
"parent_id": self.parent_id,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class DesignToken(Base, TimestampMixin, SoftDeleteMixin):
"""One custom property in one system: its name, and its value per mode.
`value_by_mode` is a JSONB map of mode -> value, e.g.
`{"base": "#f7f5ef", "dark": "#14171a"}`. Three reasons it beats a pair of
`value_light` / `value_dark` columns here:
- **Absence means one thing.** In a child system an unset mode means
"inherit"; in a root it would have to mean "not mode-dependent". With
columns those are both NULL and the resolver cannot tell them apart. With
a map, resolution is `{**parent_map, **child_map}` at every level —
one rule, no special case for roots.
- **Per-mode overrides are already real.** A palette rule in this operator's
own kit deepens one accent on light backgrounds for contrast while
leaving the dark value alone. Mode is a second override axis, not a
second column.
- **Nothing filters tokens by value in SQL.** Drift comparison resolves the
set first and compares in the client; the importer diffs in Python. The
queryability columns would buy is for a query no caller makes.
The cost is real — a third mode is data rather than schema, so the DB will
not reject a typo'd mode key. That is the trade taken.
NOT NULL with a `{}` default deliberately: a JSONB column otherwise has two
empty states (SQL NULL and JSON null) and code has to test for both.
"""
__tablename__ = "design_tokens"
__table_args__ = (
# Partial unique: a name is unique among LIVE tokens in a system, so a
# trashed token doesn't block recreating the same name. Two live rows
# named `--fs-obsidian` in one system is a duplicate definition, and the
# cascade would pick between them arbitrarily.
Index(
"uq_token_per_design_system", "design_system_id", "name",
unique=True, postgresql_where=text("deleted_at IS NULL"),
),
)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
design_system_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("design_systems.id", ondelete="CASCADE"), index=True
)
name: Mapped[str] = mapped_column(Text)
# Named for what it holds rather than the bare word `values`, which is
# reserved in SQL — the same reason `group_name` is not `group`.
value_by_mode: Mapped[dict] = mapped_column(
JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")
)
# `group` is a reserved word in SQL; `group_name` throughout — model, column
# and payload — so no layer has to remember which spelling it is on.
# Free text, not a CHECK enum: groupings are the design system's own
# vocabulary, and a whitelist would bake one install's kit into the schema.
group_name: Mapped[str | None] = mapped_column(Text, nullable=True)
purpose: Mapped[str | None] = mapped_column(Text, nullable=True)
# WHY this token is this value — a different question from `purpose`, which
# is what it is FOR. "Deliberately the same value as the primary action
# colour" is a rationale; "page background, deepest surface" is a purpose.
# Design guidance carries the first routinely and a token row had nowhere to
# put it.
rationale: Mapped[str | None] = mapped_column(Text, nullable=True)
# Literal values this token should be used INSTEAD OF, e.g. ["#fff",
# "#ffffff"] on a text-on-action token.
#
# This is how a design system records the thing a prohibition was trying to
# say. "Pure white is never text" is the shadow of a positive fact — some
# other colour IS the text colour — and a system that stores what things ARE
# has no row for a ban.
# Recording the replacement keeps the check and makes it actionable: a
# finding can name what to write instead of merely objecting.
#
# It has to be DECLARED rather than inferred, because the superseded literal
# and the token's own value are usually different colours entirely. No
# value-matching rule could ever connect them.
#
# Consumed by the source lint, not by the drift panel: these
# literals live in component CSS, which the panel cannot see and says so.
supersedes: Mapped[list] = mapped_column(
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb")
)
order_index: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
def to_dict(self) -> dict:
return {
"id": self.id,
"design_system_id": self.design_system_id,
"name": self.name,
"value_by_mode": self.value_by_mode or {},
"group_name": self.group_name,
"purpose": self.purpose,
"rationale": self.rationale,
"supersedes": self.supersedes or [],
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+177
View File
@@ -0,0 +1,177 @@
from datetime import datetime, timezone
from pgvector.sqlalchemy import Vector
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
# bge-small-en-v1.5 produces 384-dim unit-normalized vectors. The column is a
# native pgvector `vector(384)` (see migration 0067) so similarity is ranked in
# Postgres by `<=>` rather than by a full-table Python cosine scan. The semantic
# searches rank the CALLER'S in-scope chunks exactly rather than through the
# HNSW index, which ranks before it filters (see embeddings._rank_scoped).
EMBEDDING_DIM = 384
class NoteEmbedding(Base):
"""One embedding vector per CHUNK of a note (migration 0077).
The model reads at most 512 tokens, so a single whole-document vector
permanently lost everything past ~400 words. A note now stores one row per
chunk of `embeddings.chunk_document`, and a query matches the note if it
matches ANY chunk — retrieval collapses rows to best-chunk-per-note.
A short note has exactly one row (chunk_index 0) whose text is the
historical `title\\nbody` shape.
"""
__tablename__ = "note_embeddings"
note_id: Mapped[int] = mapped_column(
Integer,
ForeignKey("notes.id", ondelete="CASCADE"),
primary_key=True,
)
chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
user_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
# Exactly what this vector encodes — inspectable when a ranking surprises,
# and the hook for surfacing WHICH section matched, later.
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
# embeddings.CHUNKER_VERSION at write time. The startup backfill re-embeds
# any note whose rows carry a stale version — shape changes become a
# version bump instead of a table wipe.
chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
# embeddings.EMBEDDING_MODEL at write time — the SPACE the vector lives in.
# The column is `vector(384)`, a width and not an identity, so a
# swap to another 384-dim model writes a second geometry beside the first
# with no error, and cosine across the two is a number that means nothing.
# The version above says what text was embedded; this says in whose
# geometry. Either one moving makes the row stale, and the backfill
# re-embeds on both.
embedding_model: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
class RuleEmbedding(Base):
"""One embedding vector per CHUNK of a rule.
A SIBLING of NoteEmbedding rather than a generalisation of it, decided
deliberately. The reasoning below is the general test for when a record
type earns its own embedding table; the later siblings apply it too.
- The embedding ROW could have been made polymorphic. The SEARCH could not.
`semantic_search_notes` is a long function of Note-specific scoping —
the visibility clause, the supersession penalty, note_type/task_kind and
system filters — and a rule shares none of it. Rules scope by rulebook
ownership and project applicability instead.
- Generalising the row while still needing two searches is the worst of
both: a polymorphic key with referential integrity to neither table, on
the path every session start runs, to share four columns.
- What is genuinely common is BEHAVIOUR, not storage — get_embedding,
chunk_document, embedding_text and CHUNKER_VERSION are already free
functions and are reused as-is. Sharing those is the DRY win; sharing
the table would have been the DRY costume.
No `user_id`: NoteEmbedding carries one and its own search deliberately
ignores it (scoping on the note instead, or shared records become
unreachable). Rather than repeat a column that exists to be ignored, a
rule's reach is resolved by joining the rule.
"""
__tablename__ = "rule_embeddings"
rule_id: Mapped[int] = mapped_column(
BigInteger,
ForeignKey("rules.id", ondelete="CASCADE"),
primary_key=True,
)
chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
# Exactly what this vector encodes — inspectable when a ranking surprises.
# For a rule this is the trigger-first document, NOT the rule's `why`:
# `why` is incident narrative and drags every rule toward one centroid.
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
embedding_model: Mapped[str] = mapped_column(Text, nullable=False) # see NoteEmbedding
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
class MilestoneEmbedding(Base):
"""One embedding vector per CHUNK of a milestone.
The third sibling, for RuleEmbedding's reason: the row could be shared, the
search cannot. A milestone is scoped by its project, has no share of its
own, and is searched to answer one question — "is there already a plan for
this?" — which no note or rule search can answer, because a plan is not a
note. Without it, a roadmap written as milestones is invisible to recall,
and a later plan gets opened as a new milestone beside the one that
already describes it.
The document is the title, the one-line description and the plan body, the
parts a reader uses to recognise a plan. Derived data: the startup backfill
regenerates it, which is also how a chunker-version bump is handled.
"""
__tablename__ = "milestone_embeddings"
milestone_id: Mapped[int] = mapped_column(
Integer,
ForeignKey("milestones.id", ondelete="CASCADE"),
primary_key=True,
)
chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
embedding_model: Mapped[str] = mapped_column(Text, nullable=False) # see NoteEmbedding
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
class SystemEmbedding(Base):
"""One embedding vector per CHUNK of a System's charter.
The fourth sibling, for the same reason as the others: the row could be
shared, the search cannot. A System's `description` is a charter —
several hundred words saying what belongs in that area and what does not —
and it is the answer to "which part of this codebase does X live in".
Without it there is no semantic path to one: `list_systems` enumerates,
and `search(system_id=…)` uses a System as a FILTER over notes, so a System
could narrow a search and never be the answer to one.
"Where does this belong?" is a different question from "what prior art is
there?", which is why this is its own search rather than a note_type: a
charter competing with two thousand notes for the same top-k would be
outranked by the records filed under it, and the right answer would be
crowded out by its own contents.
The document is the name and the charter. Derived data: the startup
backfill regenerates it, which is also how a chunker-version bump is
handled.
"""
__tablename__ = "system_embeddings"
system_id: Mapped[int] = mapped_column(
Integer,
ForeignKey("systems.id", ondelete="CASCADE"),
primary_key=True,
)
chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
embedding_model: Mapped[str] = mapped_column(Text, nullable=False) # see NoteEmbedding
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
+392
View File
@@ -0,0 +1,392 @@
"""Family canon: ideas that every project on a platform shares,
and each project's recorded answer to them.
A FAMILY IDEA is not a new kind of record. It is an existing `notes` row — a
note, a snippet, a lesson — that has been given a platform scope. The note
carries the idea (when it applies, the traps, the checklist, the incidents
behind them); an optional rulebook topic carries the norms that bind; snippets
are the reference implementations, one per language. A new record type would
be one more thing reached for interchangeably with rules, processes, snippets
and design systems, and nothing here needs one.
What is shared is the IDEA, not the code. Each project implements it in its own
language; identical code is a by-product and never the test.
The tables, and the one job each does:
- `platforms` — the global catalog of what a project can be built on or ship
as. Same shape and reasoning as `canonical_systems`: no owner, so the same
word means the same thing in every project on the install.
- `project_platforms` — which platforms a project is. Membership is how
"is this in family?" stops being a judgment made per idea and becomes a
lookup.
- `family_ideas` — a note's family state: candidate, canon or retired, its
applicability test, its canon version, its linked rule topic.
- `family_idea_platforms` — the platforms an idea is for. The ONLY scope
source: a linked topic carries no platforms of its own, so the two can
never disagree. The scope decides which projects owe the idea an answer;
the topic's rules still reach a session by ordinary rule retrieval.
- `family_idea_references` — the reference implementations.
- `family_adoptions` — one row per (project, idea): the project's answer.
- `family_decisions` — the append-only log of every promotion and every
ledger change, with its reason and the earlier decisions it followed. The
agent makes these calls with no approval step, so the log is how they stay
consistent (precedent) and how a person reviews or undoes one.
"""
from datetime import datetime
from sqlalchemy import (
BigInteger, DateTime, ForeignKey, Index, Integer, Text, text,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, SoftDeleteMixin, TimestampMixin, iso
# Each tuple is the whitelist its CHECK constraint enforces (migration 0120).
# A new value means DROP + ADD CONSTRAINT in the same migration.
MEMBERSHIP_STATES = ("declared", "detected", "rejected")
IDEA_STATUSES = ("candidate", "canon", "retired")
ADOPTION_STATUSES = ("unassessed", "adopted", "variant", "exempt", "owed")
# The two adoption answers that are a departure, and so must say why.
REASONED_STATUSES = ("variant", "exempt")
DECISION_ACTIONS = ("propose", "promote", "revise", "retire", "assess", "undo")
DECIDERS = ("agent", "operator", "system")
class Platform(Base, TimestampMixin, SoftDeleteMixin):
"""What a project is built on or ships as — a runtime, a delivery channel
or a toolchain whose own behaviour causes the problems a family idea
answers.
FLAT, deliberately. A project declares several, and an idea is for
several, so "Android app and container image" needs no hierarchy to
express. Applicability narrower than a platform (an idea that only matters
to a sideloaded APK, not a store-distributed one) belongs in the idea's
`applies_when`, where a project it does not fit answers `exempt` with the
reason. Splitting platforms to carry that would grow the catalog every
time an idea got more specific.
GLOBAL, like `canonical_systems`: no owner, so a shared project inherits
the vocabulary rather than re-earning it. `slug` is the match key and the
form a backup carries, since ids are per-install.
`markers` are glob patterns over a bound repo's paths that suggest a
project is this platform (platform detection). A pattern without a slash
matches a file's basename anywhere in the tree; one with a slash matches
the repo-relative path. Empty means declare-only: some platforms leave no
reliable file behind.
"""
__tablename__ = "platforms"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(Text, nullable=False)
slug: Mapped[str] = mapped_column(Text, nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
markers: Mapped[list] = mapped_column(
JSONB, default=list, server_default=text("'[]'::jsonb"),
)
order_index: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
__table_args__ = (
# Unique among LIVE rows, so a retired entry doesn't block recreating
# the same platform (the canonical_systems convention).
Index(
"uq_platforms_slug", "slug",
unique=True, postgresql_where=text("deleted_at IS NULL"),
),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"name": self.name,
"slug": self.slug,
"description": self.description,
"markers": list(self.markers or []),
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class ProjectPlatform(Base, CreatedAtMixin):
"""A project's answer to "are you this platform?".
Three states, because detection is automatic and must not be able to
override a person:
- ``declared`` — someone said so (at inception or in settings).
- ``detected`` — a marker in a bound repo said so.
- ``rejected`` — someone said NO. Kept as a row rather than deleted, so the
next refresh does not detect it straight back. Not membership.
Only `declared` and `detected` make a project a member of the platform's
family.
"""
__tablename__ = "project_platforms"
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True,
)
platform_id: Mapped[int] = mapped_column(
Integer, ForeignKey("platforms.id", ondelete="CASCADE"), primary_key=True,
index=True,
)
# CHECK ck_project_platforms_state (migration 0120).
state: Mapped[str] = mapped_column(Text, default="declared", server_default="declared")
def to_dict(self) -> dict:
return {
"project_id": self.project_id,
"platform_id": self.platform_id,
"state": self.state,
"created_at": iso(self.created_at),
}
class FamilyIdea(Base, TimestampMixin):
"""A note's family state. Its presence is what makes the note a family
idea; the note itself is unchanged.
- ``candidate`` — recorded, not yet proven or not yet stated in platform
terms. Reaches nobody's ledger.
- ``canon`` — promoted. Every project on its platforms owes it an answer.
- ``retired`` — demoted. Kept, with its ledger, so the history reads.
`applies_when` is the applicability test every assessment starts from —
stated in platform terms, which is the first promotion criterion. A CHECK
refuses canon without one, so that criterion is the schema's to enforce
and not a sentence a session might skip.
`canon_version` moves whenever the idea's substance changes. An adoption
assessed against an older version needs rechecking — which is DERIVED by
comparing the two numbers, never stored as a flag that could go stale.
`topic_id` is the rule topic holding the idea's binding norms, if it has
any, set by `set_topic`. The topic
never carries platforms of its own; the idea's readouts list its rules,
so an assessment judges a project against them.
"""
__tablename__ = "family_ideas"
note_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True,
)
# CHECK ck_family_ideas_status (migration 0120).
status: Mapped[str] = mapped_column(Text, default="candidate", server_default="candidate")
# CHECK ck_family_ideas_canon_applies (migration 0120).
applies_when: Mapped[str | None] = mapped_column(Text, nullable=True)
canon_version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
topic_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("rulebook_topics.id", ondelete="SET NULL"), nullable=True,
)
__table_args__ = (
# One idea per topic: a topic's norms belong to one standard.
Index(
"uq_family_ideas_topic", "topic_id",
unique=True, postgresql_where=text("topic_id IS NOT NULL"),
),
Index("ix_family_ideas_status", "status"),
)
def to_dict(self) -> dict:
return {
"note_id": self.note_id,
"status": self.status,
"applies_when": self.applies_when or "",
"canon_version": self.canon_version,
"topic_id": self.topic_id,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class FamilyIdeaPlatform(Base, CreatedAtMixin):
"""A platform an idea is for. A join table with a model so the backup's
column guard can see it, and so the reverse lookup — every idea for a
platform — has an index."""
__tablename__ = "family_idea_platforms"
note_id: Mapped[int] = mapped_column(
Integer, ForeignKey("family_ideas.note_id", ondelete="CASCADE"), primary_key=True,
)
platform_id: Mapped[int] = mapped_column(
Integer, ForeignKey("platforms.id", ondelete="CASCADE"), primary_key=True,
index=True,
)
class FamilyIdeaReference(Base, CreatedAtMixin):
"""A reference implementation of an idea: a snippet a project can start
from. Explicit rather than inferred, because an owed task names "the
reference for your language", and a guess there sends a project to copy
the wrong thing. The snippet's language is read from the snippet."""
__tablename__ = "family_idea_references"
idea_id: Mapped[int] = mapped_column(
Integer, ForeignKey("family_ideas.note_id", ondelete="CASCADE"), primary_key=True,
)
snippet_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True,
index=True,
)
class FamilyAdoption(Base, TimestampMixin):
"""One project's answer to one family idea — the adoption ledger.
- ``unassessed`` — the idea reached the project and nobody has judged it.
- ``adopted`` — the project does it.
- ``variant`` — the project departs, for a reason that names a fact about
itself the canon did not account for. A preference is not a reason.
- ``exempt`` — the idea's `applies_when` is false for this project.
- ``owed`` — it applies, there is no reason to depart, and it is not done
yet. `owed_task_id` is the task filed in THIS project to do it.
`reason` is required for variant and exempt (CHECK), because those two are
the departures and the reason is the whole record. `canon_version` is the
idea version this answer was given against; NULL while unassessed.
The row is the CURRENT answer. How it got there — the reasons at each step
and the precedents each followed — is in `family_decisions`.
"""
__tablename__ = "family_adoptions"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), index=True,
)
idea_id: Mapped[int] = mapped_column(
Integer, ForeignKey("family_ideas.note_id", ondelete="CASCADE"), index=True,
)
# CHECK ck_family_adoptions_status and ck_family_adoptions_reason (0120).
status: Mapped[str] = mapped_column(Text, default="unassessed", server_default="unassessed")
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
canon_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
assessed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
# CHECK ck_family_adoptions_decided_via (0120). NULL while unassessed.
decided_via: Mapped[str | None] = mapped_column(Text, nullable=True)
owed_task_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True,
)
__table_args__ = (
Index("uq_family_adoptions_pair", "project_id", "idea_id", unique=True),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"project_id": self.project_id,
"idea_id": self.idea_id,
"status": self.status,
"reason": self.reason or "",
"canon_version": self.canon_version,
"assessed_at": iso(self.assessed_at),
"decided_via": self.decided_via,
"owed_task_id": self.owed_task_id,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class FamilyDecision(Base, CreatedAtMixin):
"""One decision about family canon — append-only.
`project_id` is NULL for a decision about the idea itself (propose,
promote, revise, retire) and set for a decision about one project's
answer (assess). `undo` reverses an earlier decision and names it in
`evidence`.
`before` and `after` are STATE snapshots — status, version, reason — and
deliberately hold no foreign ids: an id inside JSON cannot be remapped by a
restore, and would come back pointing at whatever took that number. The ids this row needs are columns.
`precedent_ids` are the earlier decisions this one followed. They are what
keeps a call consistent with the last similar one when no person approves
either, so they are a list of ids into this same table, remapped at
restore through the decision map.
"""
__tablename__ = "family_decisions"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
idea_id: Mapped[int] = mapped_column(
Integer, ForeignKey("family_ideas.note_id", ondelete="CASCADE"), index=True,
)
project_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=True, index=True,
)
# CHECK ck_family_decisions_action (0120).
action: Mapped[str] = mapped_column(Text)
reason: Mapped[str] = mapped_column(Text)
before: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
after: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
precedent_ids: Mapped[list] = mapped_column(
JSONB, default=list, server_default=text("'[]'::jsonb"),
)
# CHECK ck_family_decisions_decided_via (0120).
decided_via: Mapped[str] = mapped_column(Text, default="agent", server_default="agent")
# Who was acting — the session's user. The decider KIND is decided_via.
user_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True,
)
def to_dict(self) -> dict:
return {
"id": self.id,
"idea_id": self.idea_id,
"project_id": self.project_id,
"action": self.action,
"reason": self.reason,
"before": self.before,
"after": self.after,
"evidence": self.evidence or {},
"precedent_ids": list(self.precedent_ids or []),
"decided_via": self.decided_via,
"user_id": self.user_id,
"created_at": iso(self.created_at),
}
def to_brief(self) -> dict:
"""The decision as a reader WEIGHING it needs it: what was decided,
why — once — and the state it moved between.
Precedents and an idea's history are read in bulk, a handful per
call, and the full row repeats itself: an assessment's reason is
stored again inside `after` (and the prior one inside `before`), and
a promotion's evidence carries the whole criteria reasoning. So a
state here is its status, version and platforms only, and evidence is
the cited list alone. The full row is `to_dict`, which the
decision log serves.
"""
return {
"id": self.id,
"idea_id": self.idea_id,
"project_id": self.project_id,
"action": self.action,
"reason": self.reason,
"before": _brief_state(self.before),
"after": _brief_state(self.after),
"evidence": list((self.evidence or {}).get("evidence") or []),
"created_at": iso(self.created_at),
}
# What a decision's before/after keeps in its brief: the state, not the prose.
_BRIEF_STATE_KEYS = ("status", "canon_version", "platforms")
def _brief_state(state: dict | None) -> dict | None:
if state is None:
return None
return {k: state[k] for k in _BRIEF_STATE_KEYS if k in state}
+47
View File
@@ -0,0 +1,47 @@
from sqlalchemy import ForeignKey, Integer, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
class ForgeConnection(Base, TimestampMixin):
"""One user's read-only credential for one git forge host.
The keyring model: a user owns a set of connections and every server-side
forge read for a project runs on the PROJECT OWNER's set, resolved by the
repo's host. One row per (user, host) — the repo's host picks the
connection deterministically, so there is no "default forge" pointer to
maintain or tie-break.
`host` is derived from `base_url` at write time and stored because it is
the lookup key; the service layer keeps the two in step. The token is a
secret: it is stored sealed (services/secrets.py — read it through
services.forge_connections.connection_token), to_dict never includes it,
and no route may return it.
"""
__tablename__ = "forge_connections"
__table_args__ = (
UniqueConstraint("user_id", "host", name="uq_forge_connections_user_host"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
kind: Mapped[str] = mapped_column(Text, nullable=False)
base_url: Mapped[str] = mapped_column(Text, nullable=False)
host: Mapped[str] = mapped_column(Text, nullable=False)
token_sealed: Mapped[str] = mapped_column(Text, nullable=False)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"kind": self.kind,
"base_url": self.base_url,
"host": self.host,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+57
View File
@@ -0,0 +1,57 @@
from datetime import datetime, timezone
from sqlalchemy import ForeignKey, Integer, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, TimestampMixin, iso
class Group(Base, TimestampMixin):
__tablename__ = "groups"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
description: Mapped[str | None] = mapped_column(Text)
created_by: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="SET NULL")
)
memberships: Mapped[list["GroupMembership"]] = relationship(
"GroupMembership", back_populates="group", cascade="all, delete-orphan"
)
def to_dict(self) -> dict:
return {
"id": self.id,
"name": self.name,
"description": self.description,
"created_by": self.created_by,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class GroupMembership(Base, CreatedAtMixin):
__tablename__ = "group_memberships"
__table_args__ = (UniqueConstraint("group_id", "user_id", name="uq_gm_group_user"),)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
group_id: Mapped[int] = mapped_column(
Integer, ForeignKey("groups.id", ondelete="CASCADE"), nullable=False
)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
role: Mapped[str] = mapped_column(Text, nullable=False, default="member")
group: Mapped["Group"] = relationship("Group", back_populates="memberships")
def to_dict(self) -> dict:
return {
"id": self.id,
"group_id": self.group_id,
"user_id": self.user_id,
"role": self.role,
"created_at": iso(self.created_at),
}
+23
View File
@@ -0,0 +1,23 @@
from datetime import datetime, timezone
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin
class InvitationToken(Base, CreatedAtMixin):
__tablename__ = "invitation_tokens"
id: Mapped[int] = mapped_column(primary_key=True)
email: Mapped[str] = mapped_column(Text, nullable=False)
token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
invited_by: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
used: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
__table_args__ = (
Index("ix_invitation_tokens_token_hash", "token_hash"),
Index("ix_invitation_tokens_email", "email"),
)
+112
View File
@@ -0,0 +1,112 @@
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, Text, UniqueConstraint, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
SUGGESTED = "suggested"
CONFIRMED = "confirmed"
REJECTED = "rejected"
# CHECK ck_lesson_rule_links_state (migration 0111).
LINK_STATES = (SUGGESTED, CONFIRMED, REJECTED)
class LessonRuleLink(Base, CreatedAtMixin):
"""A lesson says it is an instance of a rule.
A lesson records one concrete situation; a rule records the binding choice
for a class of them. The link lets the rule be reached through the
situations that keep proving it, and lets a situation with lessons and no
rule be noticed. A lesson never BECOMES a rule — it points at one.
ONE ROW PER PAIR, WITH A STATE, because the operator's design forms a link
in two stages ("a sort of soft and then hard link once it's been proven"):
- ``suggested`` — evidence is accumulating that the two belong together
(they keep arriving in the same request, in distinct situations). It
carries nothing in retrieval: a suggested link that brought its rule
along would manufacture the co-surfacing it counts, and prove itself.
- ``confirmed`` — a judgment said the lesson is an instance of the rule.
The only state that changes what surfaces.
- ``rejected`` — a judgment said it is not. Kept, evidence and all, so the
pair is never proposed again and the reason stays readable.
A table rather than a list on the lesson's `data`: the link needs a foreign
key on both ends (a deleted rule must not leave a lesson pointing at
nothing), a reverse index (a rule lists its lessons), and an id remap at
restore — none of which a JSON list gives.
"""
__tablename__ = "lesson_rule_links"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
lesson_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), index=True,
)
rule_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), index=True,
)
state: Mapped[str] = mapped_column(Text, default=SUGGESTED, server_default=SUGGESTED)
# Why it was confirmed or rejected — the reasoning a later reader needs to
# decide whether it still holds, as a rule relation's `note` is.
note: Mapped[str | None] = mapped_column(Text, nullable=True)
# What the suggestion rests on (filled by the co-surfacing recorder).
# Kept after a judgment so a confirmation can be read beside its evidence.
evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
# When a judgment moved it out of `suggested`. Null while suggested.
judged_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True,
)
__table_args__ = (
UniqueConstraint("lesson_id", "rule_id", name="uq_lesson_rule_links_pair"),
)
def to_dict(self) -> dict:
return {
"lesson_id": self.lesson_id,
"rule_id": self.rule_id,
"state": self.state,
"note": self.note or "",
"evidence": self.evidence or {},
"judged_at": iso(self.judged_at),
"created_at": iso(self.created_at),
}
class LessonNoRule(Base):
"""A judgment that no rule governs a lesson's situation.
The third answer to "which rule is this an instance of?", beside naming
one and leaving it open. It is what lets an UNJUDGED lesson be told apart
from one that was looked at and found to stand alone — and a lesson with
no rule is the raw material convergence reads, so the
difference is the whole signal.
A row of its own rather than a key in the lesson's `data`: that column is
a mirror re-composed from the body on every edit (services/lessons.py),
so anything kept there that the body does not render is erased by the
next rewording — and the lesson would silently read as unjudged again.
One row per lesson. Naming a rule later deletes it: "no rule fits" and
"this rule fits" cannot both be the current answer.
"""
__tablename__ = "lesson_no_rule"
lesson_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True,
)
# Why none fits — required, because "no rule" with no reason cannot be
# re-judged when a rule is later written for the situation.
why: Mapped[str] = mapped_column(Text)
judged_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(),
)
def to_dict(self) -> dict:
return {"why": self.why, "judged_at": iso(self.judged_at)}
+35
View File
@@ -0,0 +1,35 @@
from sqlalchemy import ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import SoftDeleteMixin, TimestampMixin, iso
class Milestone(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "milestones"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"))
project_id: Mapped[int] = mapped_column(Integer, ForeignKey("projects.id", ondelete="CASCADE"))
title: Mapped[str] = mapped_column(Text, default="")
description: Mapped[str | None] = mapped_column(Text, nullable=True)
# The plan: design/intent/purpose (markdown). The milestone is the plan
# container; its steps live as first-class child tasks (milestone_id), not
# as checkboxes in this body. `description` stays the one-line summary.
body: Mapped[str | None] = mapped_column(Text, nullable=True)
status: Mapped[str] = mapped_column(Text, default="active")
order_index: Mapped[int] = mapped_column(Integer, default=0)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"project_id": self.project_id,
"title": self.title,
"description": self.description,
"body": self.body,
"status": self.status,
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+75
View File
@@ -0,0 +1,75 @@
from sqlalchemy import BigInteger, ForeignKey, Integer, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class MomentMapping(Base, CreatedAtMixin):
"""One install's correction to which actions reach which moment.
The moments are the product's vocabulary and the shipped defaults say how
the common actions reach them (`services/moment_actions.py`). What no
default can know is how THIS operator works: one delivers with a push,
another with `make ship`, a third by publishing a document. A row here is
that local knowledge, written in-session the moment a misfire is noticed.
`effect` is "add" (this action reaches this moment) or "remove" (a shipped
default that misfires here is switched off). Removal is a row rather than
an edit to the defaults because the defaults are code: an install can only
say "not here", and saying so must survive an upgrade that ships the same
default again.
Text rather than a CHECK on `effect`, for retrieval_tuning_events' reason:
the service validates it, and a constraint would buy nothing but
a migration the day a third effect is wanted.
ONE ROW PER (user, tool, match, moment), so mapping the same thing twice
is an update of the reason rather than a duplicate, and an add and a remove
of the same mapping cannot both stand.
CASCADES with the user, unlike the telemetry tables: this is the user's own
configuration, not a history that should outlive them.
"""
__tablename__ = "moment_mappings"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False,
)
# The tool as the harness names it, with any MCP server prefix stripped
# (`mcp__plugin_x__update_task` → `update_task`), so a mapping does not
# depend on what an install called its server.
tool: Mapped[str] = mapped_column(Text, nullable=False)
# "" = every call of the tool. For a tool that runs a command, how the
# command starts ("make ship"); for any other tool, `field=value` pairs.
match: Mapped[str] = mapped_column(Text, nullable=False, default="")
moment: Mapped[str] = mapped_column(Text, nullable=False)
effect: Mapped[str] = mapped_column(Text, nullable=False, default="add")
# Why — what misfired, or what this action is for here. Optional at the
# boundary: the correction is usually self-explanatory, and a required
# reason would be friction on the exact in-session fix this table exists
# to make cheap.
reason: Mapped[str] = mapped_column(Text, nullable=False, default="")
# "model" | "human", for retrieval_tuning_events' reason: both act as the
# same user, and "did I do this or did the session?" is the first question.
actor: Mapped[str] = mapped_column(Text, nullable=False, default="model")
__table_args__ = (
UniqueConstraint(
"user_id", "tool", "match", "moment", name="uq_moment_mapping",
),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"tool": self.tool,
"match": self.match,
"moment": self.moment,
"effect": self.effect,
"reason": self.reason,
"actor": self.actor,
"created_at": iso(self.created_at),
}
+217
View File
@@ -0,0 +1,217 @@
import enum
from datetime import date, datetime
from sqlalchemy import Date, DateTime, ForeignKey, Index, Integer, Text
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import SoftDeleteMixin, TimestampMixin, iso
class TaskStatus(str, enum.Enum):
todo = "todo"
in_progress = "in_progress"
done = "done"
cancelled = "cancelled"
class TaskPriority(str, enum.Enum):
none = "none"
low = "low"
medium = "medium"
high = "high"
class TaskKind(str, enum.Enum):
"""What KIND of work a task is. Mirrors CHECK notes_task_kind_check.
Every value the COLUMN may hold, including `plan`. That is deliberate:
plans became milestones in 0066, but historical plan-tasks still carry
the value and must stay readable and writable. Refusing to MINT a new
plan is a door policy (see the create/update task tools), not a
statement about what the column accepts — conflating the two would make
old rows unwritable, which is how a retired value turns into corrupt
data.
"""
work = "work"
issue = "issue"
spike = "spike"
plan = "plan"
class Note(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "notes"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=True
)
title: Mapped[str] = mapped_column(Text, default="")
body: Mapped[str] = mapped_column(Text, default="")
description: Mapped[str | None] = mapped_column(Text, nullable=True)
tags: Mapped[list[str]] = mapped_column(ARRAY(Text), default=list)
parent_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
# Provenance: the task/feature an issue arose from. Distinct from parent_id
# (sub-task hierarchy) — this is "what spawned this". Only meaningful for
# issues; nullable for every record.
arose_from_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
project_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="SET NULL"), nullable=True
)
milestone_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("milestones.id", ondelete="SET NULL"), nullable=True
)
status: Mapped[str | None] = mapped_column(Text, nullable=True)
priority: Mapped[str | None] = mapped_column(Text, nullable=True)
due_date: Mapped[date | None] = mapped_column(Date, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
recurrence_rule: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
recurrence_next_spawn_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
# The claim — which session is working this task NOW.
# Orthogonal to `status`: status is the work's state and durable, the claim
# is a session's attention and dies on read once `claim_touched_at` is past
# the lease. Semantics in services/task_claims.py.
claimed_by: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
)
claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
claim_touched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
claim_session: Mapped[str | None] = mapped_column(Text, nullable=True)
# WHAT KIND of record this is, on the note/entity axis. Task-ness is tracked
# by `status`, not here (person/place/list entity types removed 2026-07):
# note (default) — authored prose, findable by what it is ABOUT
# process — a stored procedure, synced to a client as a skill
# snippet — a reusable shape, with its structured fields mirrored in
# `data` and its trigger composed into the title (0070)
# lesson — a transferable insight, findable by WHEN IT APPLIES rather
# than by topic. Same mirror discipline as a
# snippet: the trigger lives in `data.when_to_apply` and is
# composed into the title and the head of the body, which is
# what puts it in the embedded document. A lesson is never a
# task — see `status` — because a lesson that acquired one
# would start appearing in open-work listings.
#
# DELIBERATELY UNGATED. Unlike `task_kind` there is no CHECK on this column:
# migration 0036 added it as plain Text with a server default and nothing
# has constrained it since, so there is no whitelist to expand when a kind
# is added. The vocabulary that actually decides what a reader can
# reach is `services.knowledge._FACETS`; an unrecognised value there
# resolves to a filter matching nothing, which is the intended answer to a
# typo.
note_type: Mapped[str] = mapped_column(Text, default="note", server_default="note")
# Task sub-kind — what KIND of work this is, not how it is going:
# work (default) — ships a change
# issue — corrective; something was broken (0065)
# spike — time-boxed, and its output is KNOWLEDGE rather than a change;
# it succeeds by producing an answer, and nothing ships (0091)
# plan — retired since 0066 (plans are milestones), kept in the CHECK
# so historical plan-tasks stay writable
# Only meaningful when the note is a task (status is not None); ordinary
# notes keep the 'work' default and ignore it. Orthogonal to note_type
# (which is the note/entity axis). CHECK notes_task_kind_check.
task_kind: Mapped[str] = mapped_column(Text, default="work", server_default="work")
# Queryable structured fields for typed records — currently snippets, whose
# name/language/signature/locations live here so they can be INDEXED. The
# body keeps the same facts in readable markdown and remains what gets
# embedded; this is a mirror for querying, not the source of truth for
# display — and it is DERIVED, so every path that writes a snippet's body
# rewrites it too (services/snippets.recompose_data, called from
# notes.update_note). 0070 left it NULL on existing rows and
# snippets.backfill_snippet_data filled them at startup; readers still fall
# back to parsing the body when it is absent (snippet_fields).
data: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
# The three fields that tell a CONSTRAINT apart from a NORM (milestone
# 317, migration 0092) — the same trio `rules` carries, and for the same
# reason. A norm is a decision: no truth value, changes only when its
# author changes it. A constraint asserts a fact about someone else's
# software and goes false with nobody watching. Only constraints get a
# check.
#
# `verify_with` is how to check it is still true; `expires_when` is the
# STATE that ends it, deliberately not a date — constraints expire when
# the ground moves, not on a schedule. `verified_at` NULL means never
# checked and sorts FIRST in the sweep: unexamined outranks
# examined-long-ago.
#
# These sit on `notes`, so every kind of row in this table has them, but
# only non-task, non-snippet records are OFFERED them (gated in
# services/notes.py). A task's decay is its status — a done issue records
# what happened and cannot go false — and a snippet already carries a
# richer, location-aware verdict in `data.verification`. On those rows
# these stay null, which is also what they mean.
#
# Most notes should leave all three empty. A null `verify_with` is not a
# gap; it is the marker for "this is a decision, there is nothing to go
# and check", and the sweep is only worth reading while that holds.
verify_with: Mapped[str | None] = mapped_column(Text, nullable=True)
expires_when: Mapped[str | None] = mapped_column(Text, nullable=True)
verified_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
__table_args__ = (
Index("ix_notes_tags", "tags", postgresql_using="gin"),
Index("ix_notes_status", "status"),
Index("ix_notes_title", "title"),
Index("ix_notes_user_id", "user_id"),
Index("ix_notes_project_id", "project_id"),
Index("ix_notes_milestone_id", "milestone_id"),
Index("ix_notes_note_type", "note_type"),
Index("ix_notes_arose_from_id", "arose_from_id"),
# Containment queries into `data` — e.g. which snippets name a given
# repo/path in their locations. See migration 0070.
Index("ix_notes_data_gin", "data", postgresql_using="gin"),
)
@property
def is_task(self) -> bool:
return self.status is not None
def to_dict(self) -> dict:
return {
"id": self.id,
"title": self.title,
"body": self.body,
"description": self.description,
"tags": self.tags or [],
"parent_id": self.parent_id,
"arose_from_id": self.arose_from_id,
"project_id": self.project_id,
"milestone_id": self.milestone_id,
"status": self.status,
"priority": self.priority,
"due_date": iso(self.due_date),
"started_at": iso(self.started_at),
"completed_at": iso(self.completed_at),
"recurrence_rule": self.recurrence_rule,
"recurrence_next_spawn_at": iso(self.recurrence_next_spawn_at),
"claim": _claim_state(self),
"is_task": self.is_task,
"note_type": self.note_type or "note",
"task_kind": self.task_kind,
# Serialized unconditionally, like every other field a given row
# kind may not use (recurrence, started_at, the task fields). The
# DERIVED "last_verified" label is the one that appears only when
# a check exists — a raw projection of the row should not make a
# client branch on which keys are present.
"verify_with": self.verify_with or "",
"expires_when": self.expires_when or "",
"verified_at": iso(self.verified_at),
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
def _claim_state(note: "Note") -> dict | None:
from grimoire.services.task_claims import claim_state
return claim_state(note)
+30
View File
@@ -0,0 +1,30 @@
from sqlalchemy import ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
class NoteDraft(Base, TimestampMixin):
__tablename__ = "note_drafts"
id: Mapped[int] = mapped_column(primary_key=True)
note_id: Mapped[int] = mapped_column(Integer, ForeignKey("notes.id", ondelete="CASCADE"))
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"))
proposed_body: Mapped[str] = mapped_column(Text)
original_body: Mapped[str] = mapped_column(Text)
instruction: Mapped[str] = mapped_column(Text, default="")
scope: Mapped[str] = mapped_column(Text, default="document")
def to_dict(self) -> dict:
return {
"id": self.id,
"note_id": self.note_id,
"user_id": self.user_id,
"proposed_body": self.proposed_body,
"original_body": self.original_body,
"instruction": self.instruction,
"scope": self.scope,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+70
View File
@@ -0,0 +1,70 @@
from sqlalchemy import ForeignKey, Index, Integer, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class NoteSupersession(Base, CreatedAtMixin):
"""A newer record's claim that it has overtaken an older one.
WHY THE RELATION POINTS FORWARD
The note being WRITTEN declares what it supersedes. The old record cannot
know it has been overtaken — asking it to record its own obsolescence is
asking it to predict the future. So the claim is made by the party that has
the knowledge, and the demotion is derived from the far end.
WHY A TABLE RATHER THAN A COLUMN
It is genuinely many-to-many and partial: one note may supersede parts of
several others, and a note may be overtaken piecemeal by several later ones.
Both directions are queried and neither is rare —
`superseded_id` answers the ranking question ("has this been overtaken?"),
`superseder_id` answers the record view ("what does this replace?"). An
array column on `notes` could be indexed for one and not the other.
WHAT IT MEANS, AND WHAT IT DOES NOT
A claim, never a proof. Supersession DEMOTES a record in ranked retrieval;
it does not assert the older record was wrong, and it never hides it. A note
that accurately described how something worked in June is still accurate
about June — it is just no longer the answer to "how does this work".
CASCADE IS SAFE HERE BECAUSE TRASHING IS NOT A DELETE
`trash_svc` stamps `deleted_at` (an UPDATE), so a trashed note keeps its
claims and `restore` brings them back intact. The cascade fires only on
`purge_trash`, where the row genuinely goes — and a supersession claim about
a row that no longer exists is not a fact anyone can act on.
"""
__tablename__ = "note_supersessions"
id: Mapped[int] = mapped_column(primary_key=True)
# The newer record, making the claim.
superseder_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE")
)
# The older record, demoted by it.
superseded_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE")
)
__table_args__ = (
UniqueConstraint(
"superseder_id", "superseded_id", name="uq_note_supersessions_pair"
),
# Both directions indexed — see the class docstring for why neither is
# the rare one.
Index("ix_note_supersessions_superseder", "superseder_id"),
Index("ix_note_supersessions_superseded", "superseded_id"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"superseder_id": self.superseder_id,
"superseded_id": self.superseded_id,
"created_at": iso(self.created_at),
}
+101
View File
@@ -0,0 +1,101 @@
from sqlalchemy import Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
SURFACED = "surfaced"
PULLED = "pulled"
class NoteUsageEvent(Base, CreatedAtMixin):
"""One row per time a note was SURFACED to the agent, or PULLED in full.
Answers the question RetrievalLog cannot: not "what did the ranker return
and with what scores" but "did anyone ever actually open this?" A snippet
nobody opens is not neutral — it competes for the injection budget on every
future turn and dilutes the menu — so the surfaced:pulled ratio is what
makes dead weight visible and prunable.
WHY A SEPARATE TABLE FROM RetrievalLog. RetrievalLog is one row per *call*,
keyed on the score distribution it exists to capture; folding un-scored
events into it would corrupt exactly the distribution threshold tuning reads
(see the KNOWN GAP note this closes in plugin_context.build_write_path_hint).
This table is one row per *note per event*, which is the grain the usage
readout needs and the grain RetrievalLog's JSONB `result_ids` can't be
indexed at. The two are complements: RetrievalLog tunes the threshold, this
tunes the corpus.
WHY NOT IN-SESSION CORRELATION. The original framing was "correlate
result_ids against a later get_note in the same session". There is no
session identity server-side — the MCP endpoint is stateless and hooks send
no session id — and introducing one would mean threading an opaque,
client-supplied token through every read path for a signal that does not
need it. Two independent counters answer the question without it: a note
surfaced 40 times and never pulled is dead weight regardless of which
sessions those events fell in.
Deliberately FK-free on user_id and note_id (mirrors RetrievalLog/AppLog):
telemetry outlives the row it describes, and deleting a note should not
erase the evidence that it was surfaced 40 times and never once opened.
"""
__tablename__ = "note_usage_events"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
note_id: Mapped[int] = mapped_column(Integer, nullable=False)
# 'surfaced' | 'pulled'
event: Mapped[str] = mapped_column(Text, nullable=False)
# Which surface produced it. Kept granular so the place arm and the
# semantic arm can be compared — that comparison is the whole reason the
# place arm needed logging at all.
#
# A CONVENTION, not a fixed vocabulary: `mcp_<tool>` for an agent call,
# `rest_<kind>` for a human opening a detail view, and a bare name for a
# hook or background surface ('auto_inject', 'write_path_place',
# 'write_path_semantic'). This comment deliberately does not list every
# member — such a list goes stale, naming sources nothing writes while
# omitting ones that exist, and a half-true enumeration reads as
# authoritative in exactly the way that misleads. `grep -rn record_pulled\\\|record_surfaced src/` is the
# authoritative list, and unlike a comment it cannot drift.
#
# The mcp_/rest_ split is load-bearing. "Is this dead weight?" is served by
# any pull; "was that injected line useful?" is served by AGENT pulls only,
# so never aggregate across the prefix without saying why.
source: Mapped[str] = mapped_column(Text, nullable=False)
# The project the READER was in when this happened — not the project the
# record belongs to, which is already on `notes.project_id`. The whole
# point is the comparison between the two: a record opened somewhere other
# than where it was written is the evidence that it TRANSFERRED, which is
# the claim the lesson kind rests on and the one thing
# RetrievalLog could half-answer and this table could not answer at all.
#
# Nullable, and null is ordinary rather than historical. A surfacing arm
# always knows the project it searched; a getter knows only what its caller
# passed, and a caller that passed nothing is reporting "unknown", not
# "none". Treat a null as unreported: it still counts toward "is this dead
# weight", and it cannot speak to "was it opened away from home".
project_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
__table_args__ = (
# The readout is always "these note ids, split by event" — a covering
# composite beats separate single-column indexes for it.
Index("ix_note_usage_note_event", "note_id", "event"),
Index("ix_note_usage_created_at", "created_at"),
Index("ix_note_usage_user_id", "user_id"),
# note_id first, like ix_note_usage_note_event: every readout starts
# from a set of note ids and narrows, never from a project.
Index("ix_note_usage_note_project", "note_id", "project_id"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": iso(self.created_at),
"user_id": self.user_id,
"note_id": self.note_id,
"event": self.event,
"source": self.source,
}
+33
View File
@@ -0,0 +1,33 @@
from sqlalchemy import ARRAY, ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class NoteVersion(Base, CreatedAtMixin):
__tablename__ = "note_versions"
id: Mapped[int] = mapped_column(primary_key=True)
note_id: Mapped[int] = mapped_column(Integer, ForeignKey("notes.id", ondelete="CASCADE"))
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"))
body: Mapped[str] = mapped_column(Text)
title: Mapped[str] = mapped_column(Text, default="")
tags: Mapped[list[str]] = mapped_column(ARRAY(Text), default=list)
pin_kind: Mapped[str | None] = mapped_column(Text, nullable=True)
pin_label: Mapped[str | None] = mapped_column(Text, nullable=True)
def to_dict(self, include_body: bool = True) -> dict:
d: dict = {
"id": self.id,
"note_id": self.note_id,
"user_id": self.user_id,
"title": self.title,
"tags": self.tags or [],
"pin_kind": self.pin_kind,
"pin_label": self.pin_label,
"created_at": iso(self.created_at),
}
if include_body:
d["body"] = self.body
return d
+31
View File
@@ -0,0 +1,31 @@
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, Integer, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class Notification(Base, CreatedAtMixin):
__tablename__ = "notifications"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
# project_shared | note_shared | group_added
type: Mapped[str] = mapped_column(Text, nullable=False)
payload: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"type": self.type,
"payload": self.payload,
"read_at": iso(self.read_at),
"created_at": iso(self.created_at),
}
+22
View File
@@ -0,0 +1,22 @@
from datetime import datetime, timezone
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin
class PasswordResetToken(Base, CreatedAtMixin):
__tablename__ = "password_reset_tokens"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
token_hash: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
used: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
__table_args__ = (
Index("ix_password_reset_tokens_token_hash", "token_hash"),
Index("ix_password_reset_tokens_user_id", "user_id"),
)
+62
View File
@@ -0,0 +1,62 @@
import enum
from sqlalchemy import BigInteger, ForeignKey, Integer, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import SoftDeleteMixin, TimestampMixin, iso
class ProjectStatus(str, enum.Enum):
active = "active"
paused = "paused"
completed = "completed"
archived = "archived"
class Project(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "projects"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=True)
title: Mapped[str] = mapped_column(Text, default="")
description: Mapped[str] = mapped_column(Text, default="")
goal: Mapped[str] = mapped_column(Text, default="")
status: Mapped[str] = mapped_column(Text, default="active")
color: Mapped[str | None] = mapped_column(Text, nullable=True) # hex color
# The design system this project's UI is built from, or NULL. NULL is the
# ordinary state, not a degraded one — most installs have no design system
# at all and nothing may assume one exists.
design_system_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("design_systems.id", ondelete="SET NULL"), nullable=True
)
# The per-project forge pin. NULL is the ordinary state: forge
# reads resolve against the owner's keyring by repo host. When set, the
# project's forge reads use ONLY this connection — an explicit, auditable
# choice, constrained by the service layer to a connection the project
# OWNER holds (never a collaborator's token).
forge_connection_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("forge_connections.id", ondelete="SET NULL"),
nullable=True,
)
# The inception record: what this project was decided to
# inherit, when, and through which door — {decided_at, decided_by, via,
# choices: {design_system_id, seed_systems}}. NULL means nobody has
# decided yet, and enter_project asks; the effects themselves live in
# design_system_id and the project's Systems — this is the WHY, kept so
# later surfaces can say it. See services/inception.py.
inception: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"title": self.title,
"description": self.description,
"goal": self.goal,
"status": self.status,
"color": self.color,
"design_system_id": self.design_system_id,
"forge_connection_id": self.forge_connection_id,
"inception": self.inception,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+45
View File
@@ -0,0 +1,45 @@
from sqlalchemy import ForeignKey, Integer, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
class RepoBinding(Base, TimestampMixin):
"""Maps a git repository to the Grimoire project it represents.
The SessionStart hook sends the working repo's normalized remote
(`repo_key`, e.g. ``git.fabledsword.com/bvandeusen/fabledscribe``) and the
server resolves it to a project — so the operator can work across many
repos/projects without pinning a project id in plugin config. The key is a
*stable identifier* (the git remote), not a dir-name heuristic.
"""
__tablename__ = "repo_bindings"
__table_args__ = (
UniqueConstraint("user_id", "repo_key", name="uq_repo_bindings_user_repo"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False
)
repo_key: Mapped[str] = mapped_column(Text, nullable=False)
# The branch the coverage refresh reads for this binding; NULL =
# the forge's default branch. Chosen at bind time so a dev-first project
# can have its ledger follow dev instead of waiting for the merge.
ref: Mapped[str | None] = mapped_column(Text, nullable=True)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"project_id": self.project_id,
"repo_key": self.repo_key,
"ref": self.ref,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+81
View File
@@ -0,0 +1,81 @@
from sqlalchemy import BigInteger, Boolean, Float, Index, Integer, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
ON_POINT = "on_point"
ADJACENT = "adjacent"
UNRELATED = "unrelated"
VERDICTS = (ON_POINT, ADJACENT, UNRELATED)
class RetrievalJudgment(Base, CreatedAtMixin):
"""One reviewer's verdict on one candidate of one logged retrieval call.
The relevance half the usage tables cannot hold. `note_usage_events` says
whether a line was OPENED, and because every menu line carries its matched
passage, "not opened" covers an unrelated line, a line whose passage
was enough, and a line the reader already had. Only someone reading the
query beside the line can tell those apart, so this records that reading —
with its reason, because a verdict nobody can re-read is a threshold.
Keyed to the `retrieval_logs` row it judges, by id and FK-free like every
telemetry table here. `query` is COPIED rather than joined: the verdict is
only re-readable beside the words it was judged against.
`rank` and `score` are the candidate's place when the logged query was
re-run for review, which may run past the arm's budget on purpose — the
candidates just under the cut are the ones a budget change would add.
`within_budget` says which side of the cut the line was.
`opened_after` is whether the reviewer's own agent pulled the record within
an hour of the call, read from `note_usage_events`. A correlation, not a
session join — the server has no session identity (see NoteUsageEvent) —
and None when it was not measured.
"""
__tablename__ = "retrieval_judgments"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
user_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
retrieval_log_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
# The judged call's surface, copied so the readout groups without a join.
source: Mapped[str] = mapped_column(Text, nullable=False)
query: Mapped[str | None] = mapped_column(Text, nullable=True)
record_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
# 1-based, in the review's re-run.
rank: Mapped[int] = mapped_column(Integer, nullable=False)
score: Mapped[float | None] = mapped_column(Float, nullable=True)
within_budget: Mapped[bool] = mapped_column(Boolean, nullable=False)
# One of VERDICTS. Plain Text, no CHECK, like the usage tables; the
# service refuses anything else.
verdict: Mapped[str] = mapped_column(Text, nullable=False)
reason: Mapped[str] = mapped_column(Text, nullable=False)
opened_after: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
__table_args__ = (
# One verdict per reviewer per line; judging it again replaces it.
UniqueConstraint(
"retrieval_log_id", "record_id", "user_id",
name="uq_retrieval_judgment_line",
),
Index("ix_retrieval_judgments_source_created", "source", "created_at"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": iso(self.created_at),
"user_id": self.user_id,
"retrieval_log_id": self.retrieval_log_id,
"source": self.source,
"query": self.query,
"record_id": self.record_id,
"rank": self.rank,
"score": self.score,
"within_budget": self.within_budget,
"verdict": self.verdict,
"reason": self.reason,
"opened_after": self.opened_after,
}
+102
View File
@@ -0,0 +1,102 @@
from datetime import datetime, timezone
from sqlalchemy import Boolean, DateTime, Float, Index, Integer, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import iso
class RetrievalLog(Base):
"""One row per semantic-retrieval call, for KB-injection tuning.
Captures what a query asked for, what came back, and the score
distribution of the results — the empirical basis for tuning the
similarity threshold and top-k per surface. `result_ids` holds the ranked
hits (id + score + rank) so a later pass can correlate "what we surfaced"
against "what the agent then fetched/referenced".
Deliberately FK-free on user_id (mirrors AppLog): telemetry should outlive
the row it describes, and a deleted user shouldn't cascade away history.
"""
__tablename__ = "retrieval_logs"
id: Mapped[int] = mapped_column(primary_key=True)
# Declared here rather than via CreatedAtMixin on purpose: the composite
# index below orders on `created_at.desc()`, which needs the column object
# in this class body — a mixin's column is not in scope there.
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Retrieval surface: 'mcp_search' | 'rest_search' | 'auto_inject' | ...
source: Mapped[str] = mapped_column(Text, nullable=False)
query: Mapped[str | None] = mapped_column(Text, nullable=True)
# Effective parameters actually used for this call.
threshold: Mapped[float | None] = mapped_column(Float, nullable=True)
limit_n: Mapped[int | None] = mapped_column(Integer, nullable=True)
project_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# The content-type filter as passed to semantic_search_notes: True=tasks,
# False=notes, NULL=any.
is_task: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# How many scored hits this call DROPPED because the session had already
# been shown them. NULLABLE, and the null is load-bearing: it means "this
# surface does not report suppression", which must not read as "nothing was
# suppressed". `result_count == 0` alone conflates two different events —
# the ranker found nothing above threshold, and the ranker found something
# the reader already had — and only the first says a threshold is too high.
# Reading a zero as a ranker decline sends a reader to move a floor that
# was never involved; an unmeasured value that renders as 0 is the same mistake with a nicer
# face, so surfaces that filter INSIDE the search leave this null.
suppressed_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
top_score: Mapped[float | None] = mapped_column(Float, nullable=True)
min_score: Mapped[float | None] = mapped_column(Float, nullable=True)
# The best score the ranker COULD have offered, before the threshold — as
# against `top_score`, which is the best it DID offer. They are equal on
# any call that returned something, and only this one exists on a call
# that returned nothing, which is the only place a bar can be judged from
# Null means the caller did not measure it, never "nothing was
# close": a 0.0 there would read as a corpus with no relevant records at
# all, which is an artifact standing in for a measurement.
best_available_score: Mapped[float | None] = mapped_column(Float, nullable=True)
# WHICH record scored that, so a reader can judge what the bar refused
# rather than only how close it came. Written from the same ranked
# candidate as the score above — the two describing different records would
# be worse than no id at all, because it invites judging the wrong one.
#
# Not a foreign key on purpose: this table spans record types (the rule arms
# store rule ids, the note arms store note ids) and `source` is what says
# which, exactly as `result_ids` has always worked.
best_available_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# [{"id": int, "score": float, "rank": int}, ...], highest-first.
result_ids: Mapped[list] = mapped_column(JSONB, nullable=False, default=list)
duration_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
__table_args__ = (
Index("ix_retrieval_logs_created_at", "created_at"),
Index("ix_retrieval_logs_user_id", "user_id"),
Index("ix_retrieval_logs_source", "source"),
Index("ix_retrieval_logs_source_created_at", "source", created_at.desc()),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": iso(self.created_at),
"user_id": self.user_id,
"source": self.source,
"query": self.query,
"threshold": self.threshold,
"limit_n": self.limit_n,
"project_id": self.project_id,
"is_task": self.is_task,
"result_count": self.result_count,
"suppressed_count": self.suppressed_count,
"top_score": self.top_score,
"min_score": self.min_score,
"result_ids": self.result_ids,
"duration_ms": self.duration_ms,
}
+117
View File
@@ -0,0 +1,117 @@
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
class RetrievalTuningEvent(Base):
"""One row per change to a retrieval surface's floor or budget.
WHY A TABLE AND NOT JUST THE SETTING
The setting holds the number. It cannot hold the argument, and the argument
is the interesting part: the model moves these values most of the time,
without being asked each time. Delegation like that is only safe if it leaves a trail the
operator can read afterwards — what changed, from what to what, and on what
evidence — and can disagree with.
So `reason` is not decoration and not optional. It is the guardrail: a model
that has to state why is a model that has to have looked, and a reason that
turns out to be wrong is a reason someone can point at. A bare number in a
settings row records that something moved and nothing about whether it
should have.
WHY `actor` IS NOT JUST `user_id`
Both a person and a model act as the same user — the MCP tools run under the
operator's own id — so the id cannot distinguish them. That distinction is
exactly what an operator scanning this table wants first: "did I do this, or
did the session?" A change the operator made themselves needs no review; one
made on their behalf might.
FK-FREE ON user_id, like `RetrievalLog` and `AppLog` above it: the history of
how a surface was tuned should outlive the account that tuned it, and a
deleted user must not cascade away the record of why a number is where it is.
"""
__tablename__ = "retrieval_tuning_events"
id: Mapped[int] = mapped_column(primary_key=True)
# Declared here rather than via CreatedAtMixin because the index below
# orders on `created_at.desc()`, which needs the column object in this class
# body — a mixin's column is not in scope there.
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# The surface's registry name, which is also its `retrieval_logs.source`.
# That identity is what lets a reader put a change next to what the change
# did; `tests/test_retrieval_surfaces.py` asserts it rather than trusting it.
surface: Mapped[str] = mapped_column(Text, nullable=False)
# "floor" or "budget". Text rather than an enum because a new CHECK value is
# a migration, and this is a young surface that may well grow a
# third tunable — a constraint bought nothing here except a future migration.
dial: Mapped[str] = mapped_column(Text, nullable=False)
# Both stored as float so one pair of columns serves both dials; a budget is
# a whole number and reads back as one. `old_value` is nullable because the
# first change to a surface has no stored predecessor — it moved off the
# shipped starting point, which is a different event from moving off a value
# somebody chose, and worth being able to tell apart.
old_value: Mapped[float | None] = mapped_column(Float, nullable=True)
new_value: Mapped[float] = mapped_column(Float, nullable=False)
# "model" | "human". See the class docstring: the user id cannot tell these
# apart, and the difference is the first thing a reviewer wants.
actor: Mapped[str] = mapped_column(Text, nullable=False, default="model")
# REQUIRED at the service boundary, not merely non-null here. A column that
# is technically non-null is satisfied by "", which is how a required field
# becomes a formality; the service refuses a blank one.
reason: Mapped[str] = mapped_column(Text, nullable=False)
# WHAT SPACE THIS NUMBER WAS MEASURED IN. A floor is a cosine
# distance in one embedding model's geometry, over documents cut one
# particular way — change either and the number describes something that
# no longer exists.
#
# On the EVENT rather than beside the setting, because this table already
# holds one row per change and `current_settings` already reads the latest
# per dial. A parallel stamp row next to the value would be a second place
# to keep in sync, and the two disagreeing is worse than neither.
#
# Both NULLABLE for the rows written before this step: those were measured
# under something, but claiming to know which would be inventing a fact.
# Null here means "unstamped", which is a different and honest answer from
# naming a model that may be wrong.
embedding_model: Mapped[str | None] = mapped_column(Text, nullable=True)
shape_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
__table_args__ = (
# The only read this table has: one surface's history, newest first.
Index(
"ix_retrieval_tuning_surface_created",
"surface",
created_at.desc(),
),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if self.created_at else None,
"surface": self.surface,
"dial": self.dial,
"old_value": self.old_value,
"new_value": self.new_value,
"actor": self.actor,
"reason": self.reason,
# Null on rows written before the stamp existed, and rendered as
# "unstamped" rather than guessed at — see the column comments.
"embedding_model": self.embedding_model,
"shape_version": self.shape_version,
}
@@ -0,0 +1,87 @@
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, ForeignKey, Text, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
from grimoire.models.lesson_rule_link import CONFIRMED, LINK_STATES, REJECTED, SUGGESTED
# The same three states as a lesson's link to a rule, for the same reasons —
# re-exported so a reader of this table need not know where they were first
# named. CHECK ck_rule_moment_judgments_state (migration 0118).
JUDGMENT_STATES = LINK_STATES
__all__ = [
"CONFIRMED", "JUDGMENT_STATES", "NO_MOMENT", "REJECTED", "SUGGESTED",
"SOURCES", "RuleMomentJudgment",
]
# The moment recorded for "this rule is about WHAT, not WHEN": no moment in
# the catalog is when it applies, and it is right to reach it by meaning. A
# row rather than an absence, for the reason LessonNoRule is one — a rule
# looked at and found to have no moment must not read as one nobody looked at,
# or every pass over the corpus would propose it again.
NO_MOMENT = ""
# Where a judgment came from. `pass` — an agent read the rule and proposed;
# `signal` — the rule kept being opened just after a moment fired; `edit` — a
# person changed the rule's moments directly, which is a judgment too.
SOURCES = ("pass", "signal", "edit")
class RuleMomentJudgment(Base, CreatedAtMixin):
"""Whether a rule belongs on a moment, and who said so.
A mount (`rule_moments`) is the current answer; this is the reasoning
behind it and the answers that are NOT mounts. One row per (rule, moment):
- ``suggested`` — proposed by a pass, or by repeated opens after the
moment fired. Delivers nothing: a suggestion that mounted itself would
manufacture the opens it counts.
- ``confirmed`` — a judgment put the rule on the moment. The mount exists.
- ``rejected`` — a judgment said it does not belong there, kept with its
reason so the pair is never proposed again, by either source.
Moment ``""`` (NO_MOMENT), confirmed, is "no moment fits this rule".
"""
__tablename__ = "rule_moment_judgments"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
rule_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), index=True,
)
moment: Mapped[str] = mapped_column(Text)
state: Mapped[str] = mapped_column(Text, default=SUGGESTED, server_default=SUGGESTED)
source: Mapped[str] = mapped_column(Text, default="pass", server_default="pass")
# Why — for a suggestion, what the proposer read in the rule; for a
# judgment, why it was confirmed or rejected.
note: Mapped[str | None] = mapped_column(Text, nullable=True)
# The co-occurrence evidence (signal source), in lesson_rules' shape.
evidence: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
# The other direction (migration 0119): sessions saying this
# MOUNTED rule arrived at the moment and did not apply. lesson_rules'
# evidence shape keyed per day, plus the reasons given and the actions
# that reached the moment; `kept_at` once the operator kept the mount.
misfire: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
judged_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True,
)
__table_args__ = (
UniqueConstraint("rule_id", "moment", name="uq_rule_moment_judgments_pair"),
)
def to_dict(self) -> dict:
return {
"rule_id": self.rule_id,
"moment": self.moment,
"state": self.state,
"source": self.source,
"note": self.note or "",
"evidence": self.evidence or {},
"misfire": self.misfire or {},
"judged_at": iso(self.judged_at),
"created_at": iso(self.created_at),
}
+126
View File
@@ -0,0 +1,126 @@
from sqlalchemy import BigInteger, Index, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
SURFACED = "surfaced"
PULLED = "pulled"
# The outcome half. Without these, a rule that was read and then ignored is
# indistinguishable from one that was read and obeyed; these are the two
# events that can tell them apart.
#
# There is deliberately NO third value for "read and ignored". That state is
# real and is the point of measuring outcomes, but it cannot be reported: an
# agent that knew it was ignoring a rule would not be ignoring it. It is
# DERIVED — a pull with no outcome — and an enum member for it would collect
# nothing while reading as though it had measured something.
APPLIED = "applied"
DEPARTED = "departed"
OUTCOMES = (APPLIED, DEPARTED)
class RuleUsageEvent(Base, CreatedAtMixin):
"""One row per time a rule was SURFACED to the agent, PULLED in full, or
ACTED ON — applied, or departed from with a stated reason.
The rule-side sibling of `note_usage_events`, beside `rule_embeddings`
and `rule_versions` — each exists because machinery built for notes gives
rules the weaker version of it.
WHY RULES NEED THEIR OWN AND CANNOT SHARE THE NOTE TABLE. Not squeamishness
about a polymorphic column — the row shares no note-specific fields and the
aggregate readout is the same shape, which is the strongest case for
sharing there is. What decides it is IDENTITY AT RESTORE. A
note id and a rule id are different namespaces resolved through different
maps, and `note_usage_events`'s importer maps `note_id` through
`note_id_map` and drops what does not resolve. A rule id parked in that
column would come back from a backup silently reattached to whatever note
happened to take that number — telemetry that is not merely lost but wrong,
and wrong in a way nothing downstream could detect.
WHAT THIS MEASURES. A standing-rule arm's usefulness cannot be observed
from its scores: an arm can fire on every call, clearing its threshold
every time, and say nothing about whether it helped. `retrieval_logs`
gives it scores; scores say what the ranker thought, never
whether the hint landed. Without a pull counter no install can tune the arm
from evidence, only from the shape of a histogram.
Deliberately FK-FREE on `rule_id` and `user_id`, matching `note_usage_events`,
`retrieval_logs` and `app_logs` — and diverging from `rule_versions`, which
does carry FKs. The difference is what the row is FOR: a version is part of
a rule's history and dies with it, while telemetry outlives the row it
describes. Deleting a rule must not erase the evidence that it was surfaced
forty times and opened never, because that evidence is precisely the case
for having deleted it.
Cells left deliberately empty: no share ACL — rules
have none of their own; no soft delete — nothing restores a telemetry row,
and the table is append-only; no embedding — an event is not a document.
"""
__tablename__ = "rule_usage_events"
# BigInteger throughout, where the note twin uses Integer. `rule_id` has to
# be, since `rules.id` is BigInteger — and once one column is, matching the
# rest costs nothing and keeps the row uniform. A high-churn append-only
# telemetry table is a poor place to discover an id ceiling.
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
user_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
rule_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
# 'surfaced' | 'pulled' | 'applied' | 'departed'
#
# Plain Text with no CHECK, as created in 0094 — which is why 0106 added
# the outcome pair without a DROP/ADD migration — that step is for
# CHECK-whitelisted columns and this is not one. Said here as well as in
# the migration because this is where the next person adding a value will
# look first.
event: Mapped[str] = mapped_column(Text, nullable=False)
# Which surface produced it. A CONVENTION, not a fixed vocabulary, and the
# note twin's comment explains why this one deliberately does not enumerate
# its members: such a list goes stale, naming a source nothing writes while
# omitting ones that exist, and a half-true enumeration reads as
# authoritative in exactly the way that misleads.
# `grep -rn record_rule_pulled\|record_rule_surfaced src/` is the
# authoritative list, and unlike a comment it cannot drift.
#
# The mcp_/rest_ prefix split is load-bearing here for the same reason it is
# for notes, and more so: "is this rule dead weight?" is served by any pull,
# but "did that injected hint land?" — the question this arm exists to
# answer — is served by AGENT pulls only. Never aggregate across the prefix
# without saying why.
source: Mapped[str] = mapped_column(Text, nullable=False)
# The WHY of a departure, and the reason the outcome pair is not simply
# two more bare event strings. A departure stripped of its reason reads
# back as a miss, so the two states this table exists to separate would
# collapse again one layer down — in the readout, where nobody would see
# it happen.
#
# Nullable because `applied` needs no argument. Following a rule is the
# unremarkable case; demanding prose for it would make the cheap event
# expensive, and an expensive event is one that stops being recorded.
# Empty on a `surfaced` or `pulled` row, which nobody asks a reason of.
detail: Mapped[str | None] = mapped_column(Text, nullable=True)
__table_args__ = (
# Every readout is "these rule ids, split by event" — a covering
# composite beats separate single-column indexes for it.
Index("ix_rule_usage_rule_event", "rule_id", "event"),
Index("ix_rule_usage_created_at", "created_at"),
Index("ix_rule_usage_user_id", "user_id"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": iso(self.created_at),
"user_id": self.user_id,
"rule_id": self.rule_id,
"event": self.event,
"source": self.source,
"detail": self.detail,
}
+93
View File
@@ -0,0 +1,93 @@
from datetime import datetime
from sqlalchemy import BigInteger, ForeignKey, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class RuleVersion(Base, CreatedAtMixin):
"""One snapshot of a rule's text, taken before an edit overwrote it.
THE SIBLING OF `note_versions`. Rules BIND BEHAVIOUR on every session that
loads them; without a history, an edit destroys what the rule used to say,
and the more consequential record has the weaker protection.
`CreatedAtMixin`, not `TimestampMixin`: a version is an EVENT. It is
written once and never updated, so an `updated_at` on it would be a column
that can only ever lie.
WHAT IS DELIBERATELY DIFFERENT FROM NoteVersion:
- `user_id` is the ACTOR — who made the edit — where NoteVersion's is the
owner, because `update_note` passes an owner-scoped id. For an audit
trail over a binding instruction, "who changed this" is the question
being asked, and a rule is editable by anyone with rulebook access.
- No `pin_kind` / `pin_label`. Those exist so a note's version can survive
autosave pruning. Nothing prunes here, so a pin would protect a row that
was never at risk.
- TEXT ONLY. A rule's Systems and its typed relations are edges with their
own lifecycle; folding them in would make one word, "version", mean two
different things — the rule's wording, and the rule's place in the
graph. `verified_at` is likewise absent: it is a stamp about a check,
not a property of the text, and the snapshot is taken before
update_rule clears it precisely so the history shows the check that was
in force when this wording was written.
"""
__tablename__ = "rule_versions"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
rule_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), index=True
)
# The actor. SET NULL rather than CASCADE: deleting a user must not erase
# the history of the rules they edited — the edit still happened, and the
# rule is still binding because of it. NoteVersion cascades because a
# note's versions belong to its owner; a rule's belong to the rule.
user_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
)
title: Mapped[str] = mapped_column(Text, default="")
statement: Mapped[str] = mapped_column(Text, default="")
why: Mapped[str | None] = mapped_column(Text, nullable=True)
how_to_apply: Mapped[str | None] = mapped_column(Text, nullable=True)
when_to_apply: Mapped[str | None] = mapped_column(Text, nullable=True)
# Carried so that a change of FORCE leaves a trace. `record_if_changed`
# snapshots only the fields a version holds, so a kind omitted here would
# make "this stopped binding" the one edit with no history behind it.
# NULL means a version older than migration 0098 — not recorded, never
# "was a rule".
kind: Mapped[str | None] = mapped_column(Text, nullable=True)
verify_with: Mapped[str | None] = mapped_column(Text, nullable=True)
expires_when: Mapped[str | None] = mapped_column(Text, nullable=True)
def to_dict(self, include_text: bool = True) -> dict:
"""The row. `include_text=False` gives the listing form.
A rule's `statement` and `why` run to thousands of characters — rule
149's `why` alone is longer than most notes — so a history LIST that
carried every field would be unreadable and expensive. The listing
answers "when, and by whom"; opening one answers "and what did it
say". Same split NoteVersion makes with `include_body`.
"""
out: dict = {
"id": self.id,
"rule_id": self.rule_id,
"user_id": self.user_id,
"title": self.title,
"created_at": iso(self.created_at),
}
if include_text:
out.update({
"statement": self.statement,
"why": self.why or "",
"how_to_apply": self.how_to_apply or "",
"when_to_apply": self.when_to_apply or "",
"kind": self.kind or "",
"verify_with": self.verify_with or "",
"expires_when": self.expires_when or "",
})
return out
+253
View File
@@ -0,0 +1,253 @@
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger, Boolean, Column, DateTime, ForeignKey, Index, Integer, Table,
Text, UniqueConstraint, text,
)
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, SoftDeleteMixin, TimestampMixin, iso
class Rulebook(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "rulebooks"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
owner_user_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("users.id", ondelete="CASCADE")
)
title: Mapped[str] = mapped_column(Text)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
def to_dict(self) -> dict:
return {
"id": self.id,
"owner_user_id": self.owner_user_id,
"title": self.title,
"description": self.description or "",
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class RulebookTopic(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "rulebook_topics"
# Partial unique: a title is unique among LIVE topics in a rulebook, so a
# trashed topic doesn't block recreating/restoring the same title.
__table_args__ = (
Index(
"uq_topic_per_rulebook", "rulebook_id", "title",
unique=True, postgresql_where=text("deleted_at IS NULL"),
),
)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
rulebook_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rulebooks.id", ondelete="CASCADE")
)
title: Mapped[str] = mapped_column(Text)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
order_index: Mapped[int] = mapped_column(Integer, default=0)
def to_dict(self) -> dict:
return {
"id": self.id,
"rulebook_id": self.rulebook_id,
"title": self.title,
"description": self.description or "",
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class Rule(Base, TimestampMixin, SoftDeleteMixin):
__tablename__ = "rules"
# Partial unique: title unique among LIVE rules in a topic (soft-deleted
# rules don't block recreating/restoring the same title).
__table_args__ = (
Index(
"uq_rule_per_topic", "topic_id", "title",
unique=True, postgresql_where=text("deleted_at IS NULL"),
),
)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
# Exactly one of topic_id / project_id is set — enforced by CHECK
# constraint ck_rule_topic_xor_project (migration 0059).
topic_id: Mapped[int | None] = mapped_column(
BigInteger,
ForeignKey("rulebook_topics.id", ondelete="CASCADE"),
nullable=True,
)
project_id: Mapped[int | None] = mapped_column(
BigInteger,
ForeignKey("projects.id", ondelete="CASCADE"),
nullable=True,
index=True,
)
title: Mapped[str] = mapped_column(Text)
statement: Mapped[str] = mapped_column(Text)
# WHEN this rule applies — the trigger, not the instruction. Required of
# new rules at the service layer and nullable here, because rules written
# before migration 0088 have none and a migration cannot invent one.
# It carries three jobs at once: it is the readable form of the canon tag,
# the half of the document that makes a rule findable by meaning, and —
# with no always-on tier — the ONLY thing that decides whether a rule ever reaches a session at all. A rule
# with no trigger is not a quiet rule, it is an unreachable one.
when_to_apply: Mapped[str | None] = mapped_column(Text, nullable=True)
# WHAT KIND of instruction this is — the only axis on a rule; delivery
# belongs entirely to retrieval.
# `rule` must be FOLLOWED: ignoring it breaks something or crosses a
# boundary. `preference` is how this person wants work DONE: ignoring it
# costs consistency, not correctness.
#
# The second half is what makes it a kind rather than a softer label — a
# preference is expected to CHANGE as the work teaches it, and the agent
# updates it in the ordinary course of working, where a rule waits for its
# author. So one column decides two behaviours: whether create's approval
# gate fires, and which voice the injected line speaks in.
#
# Lives here and not in its own table because a preference needs exactly
# what a rule has and a note does not — a trigger column, a
# trigger-dominated document, ownership-scoped search, the retrieval arms,
# relations, and `rule_versions`, which is where its drift is recorded.
# Defaults to `rule` so nothing changes force on upgrade.
# CHECK ck_rules_kind (migration 0098).
kind: Mapped[str] = mapped_column(Text, default="rule", server_default="rule")
why: Mapped[str | None] = mapped_column(Text, nullable=True)
how_to_apply: Mapped[str | None] = mapped_column(Text, nullable=True)
# The three fields that tell a CONSTRAINT apart from a NORM (milestone
# 312). A norm is a decision — no truth value, changes only when its
# author changes it. A constraint asserts a fact about someone else's
# software, and goes false with nobody watching: every stale rule the
# 307 audit found was one, and no norm had rotted.
#
# `verify_with` is how to check the rule is still true; `expires_when` is
# the STATE that ends it, deliberately not a date — constraints expire
# when the ground moves, not on a schedule. `verified_at` NULL means
# never checked, and sorts FIRST in the sweep: unexamined outranks
# examined-long-ago.
#
# Most rules should leave all three empty. A null `verify_with` is not a
# gap — it is the marker for "this is a decision, there is nothing to go
# and check," and the signal is only worth reading while that stays true.
verify_with: Mapped[str | None] = mapped_column(Text, nullable=True)
expires_when: Mapped[str | None] = mapped_column(Text, nullable=True)
verified_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
# The record that caused this rule — the edge notes and tasks already
# have. A `why` can name its origin in prose; this is that link as a
# field, so it survives a rewording of the paragraph.
arose_from_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("notes.id", ondelete="SET NULL"), nullable=True
)
order_index: Mapped[int] = mapped_column(Integer, default=0)
def to_dict(self) -> dict:
return {
"id": self.id,
"topic_id": self.topic_id,
"project_id": self.project_id,
"title": self.title,
"statement": self.statement,
"when_to_apply": self.when_to_apply or "",
# Unconditional, unlike the `if present` keys below. A reader
# deciding how much force a record carries must never infer it
# from an ABSENT key: "no kind field" and "kind is rule" would be
# the same payload, and that equivalence is the defect shape this
# codebase keeps re-encountering. Twenty bytes buys an answer that
# cannot be misread.
"kind": self.kind or "rule",
"why": self.why or "",
"how_to_apply": self.how_to_apply or "",
"verify_with": self.verify_with or "",
"expires_when": self.expires_when or "",
"verified_at": iso(self.verified_at),
"arose_from_id": self.arose_from_id,
"order_index": self.order_index,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
# Which global AREA a rule is about. Points at the canonical catalog, NEVER at
# a project's `systems` row: a rule that spans projects cannot be chained to
# one project's vocabulary.
rule_systems = Table(
"rule_systems",
Base.metadata,
Column("rule_id", BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True),
Column("canonical_id", Integer, ForeignKey("canonical_systems.id", ondelete="CASCADE"), primary_key=True),
Column("created_at", DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)),
)
# Which MOMENTS of work a rule arrives at. A rule mounted on a
# moment is delivered whenever that moment happens, by lookup and with no
# score — the route for a rule that is about WHEN something is done rather
# than what it is about, which semantic retrieval structurally misses. The
# moment is a catalog NAME (`services/moments.py`), not a key: the catalog is
# code, so there is nothing to point a foreign key at, and the service refuses
# an unknown name before it is stored.
rule_moments = Table(
"rule_moments",
Base.metadata,
Column("rule_id", BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True),
Column("moment", Text, primary_key=True),
Column("created_at", DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)),
)
class RuleRelation(Base, CreatedAtMixin):
"""A typed edge between two rules. Each kind exists because without it a
rulebook needs a workaround.
- ``co_surfaces`` — these fail together, so they must arrive together.
Without it, the only way to guarantee that is to merge them into one
row, because either rule could surface without the other.
Symmetric in meaning; stored once and read both ways.
- ``overrides`` — this rule supersedes that one for its scope. Without
it, an override is a parallel rule that drifts from its parent.
- ``elaborates`` — this rule adds local specifics to that one; surfacing
the parent brings the addendum with it.
``note`` records WHY the edge was drawn, for the same reason a rule
carries `why`: a later reader deciding whether it still holds needs the
reasoning, not just the fact.
"""
__tablename__ = "rule_relations"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
from_rule_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), index=True
)
to_rule_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("rules.id", ondelete="CASCADE"), index=True
)
# CHECK ck_rule_relations_kind (migration 0088).
kind: Mapped[str] = mapped_column(Text)
note: Mapped[str | None] = mapped_column(Text, nullable=True)
__table_args__ = (
UniqueConstraint("from_rule_id", "to_rule_id", "kind", name="uq_rule_relations_edge"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"from_rule_id": self.from_rule_id,
"to_rule_id": self.to_rule_id,
"kind": self.kind,
"note": self.note or "",
"created_at": iso(self.created_at),
}
# A rule's home is its scope: a rule in a rulebook topic is global, a rule on
# a project applies to that project, and retrieval reads that directly. There
# are no per-project subscriptions or suppressions (migration 0101 dropped
# those tables); a project that departs from a global rule writes a project
# rule with an `overrides` relation.
+17
View File
@@ -0,0 +1,17 @@
from sqlalchemy import ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
class Setting(Base):
__tablename__ = "settings"
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
key: Mapped[str] = mapped_column(Text, primary_key=True)
value: Mapped[str] = mapped_column(Text, default="")
def to_dict(self) -> dict:
return {"key": self.key, "value": self.value}
+79
View File
@@ -0,0 +1,79 @@
from sqlalchemy import CheckConstraint, ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
class ProjectShare(Base, TimestampMixin):
__tablename__ = "project_shares"
__table_args__ = (
CheckConstraint(
"(shared_with_user_id IS NOT NULL) != (shared_with_group_id IS NOT NULL)",
name="ck_ps_exclusive_target",
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False
)
shared_with_user_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE")
)
shared_with_group_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("groups.id", ondelete="CASCADE")
)
permission: Mapped[str] = mapped_column(Text, nullable=False)
invited_by: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id"), nullable=False
)
def to_dict(self) -> dict:
return {
"id": self.id,
"project_id": self.project_id,
"shared_with_user_id": self.shared_with_user_id,
"shared_with_group_id": self.shared_with_group_id,
"permission": self.permission,
"invited_by": self.invited_by,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class NoteShare(Base, TimestampMixin):
__tablename__ = "note_shares"
__table_args__ = (
CheckConstraint(
"(shared_with_user_id IS NOT NULL) != (shared_with_group_id IS NOT NULL)",
name="ck_ns_exclusive_target",
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
note_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE"), nullable=False
)
shared_with_user_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE")
)
shared_with_group_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("groups.id", ondelete="CASCADE")
)
permission: Mapped[str] = mapped_column(Text, nullable=False)
invited_by: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id"), nullable=False
)
def to_dict(self) -> dict:
return {
"id": self.id,
"note_id": self.note_id,
"shared_with_user_id": self.shared_with_user_id,
"shared_with_group_id": self.shared_with_group_id,
"permission": self.permission,
"invited_by": self.invited_by,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+94
View File
@@ -0,0 +1,94 @@
from sqlalchemy import ForeignKey, Index, Integer, Text, UniqueConstraint, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, SoftDeleteMixin, TimestampMixin, iso
class System(Base, TimestampMixin, SoftDeleteMixin):
"""A per-project, reusable, self-describing subsystem/area.
Any record (note, task, or issue) can be associated with one or more
systems via record_systems, so research notes, build-tasks, and corrective
work line up under the same area — and recurring problem-areas become
filterable. Self-describing (name + description) because a bare name is
never enough to convey what a system is or how it's used.
"""
__tablename__ = "systems"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
Integer, ForeignKey("users.id", ondelete="CASCADE")
)
project_id: Mapped[int] = mapped_column(
Integer, ForeignKey("projects.id", ondelete="CASCADE")
)
name: Mapped[str] = mapped_column(Text, default="", server_default="")
# The GLOBAL area this local System is an instance of.
# Nullable and SET NULL on purpose: the catalog is a convergence aid, not a
# gate — an unmapped System is fully usable, and retiring a canonical entry
# must never take a project's System with it. The local `name` is NEVER
# rewritten to match the canonical one; this column is the join key, and
# the name stays whatever the project calls the area.
canonical_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("canonical_systems.id", ondelete="SET NULL"), nullable=True
)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
color: Mapped[str | None] = mapped_column(Text, nullable=True)
# active | archived — systems accumulate; archive rather than delete.
status: Mapped[str] = mapped_column(Text, default="active", server_default="active")
order_index: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
# The files that ARE this area, as globs relative to the repo root
# The description says what the area is FOR; these say
# where it lives, so a command or edit touching those files can be
# resolved to the System — and its rulings — without a similarity search.
# NOT NULL with a `[]` default for the reason DesignToken.supersedes gives:
# a nullable list has two empties, and every reader must handle both.
path_patterns: Mapped[list] = mapped_column(
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb")
)
__table_args__ = (
Index("ix_systems_project_id", "project_id"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"user_id": self.user_id,
"project_id": self.project_id,
"name": self.name,
"canonical_id": self.canonical_id,
"description": self.description,
"color": self.color,
"status": self.status,
"order_index": self.order_index,
"path_patterns": list(self.path_patterns or []),
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
class RecordSystem(Base, CreatedAtMixin):
"""M2M association: a record (notes.id — any note/task/issue) <-> a System.
Mutable over time; uniqueness keeps a record from linking the same system
twice. Cascades on both sides, so deleting a record or a system clears its
associations.
"""
__tablename__ = "record_systems"
id: Mapped[int] = mapped_column(primary_key=True)
note_id: Mapped[int] = mapped_column(
Integer, ForeignKey("notes.id", ondelete="CASCADE")
)
system_id: Mapped[int] = mapped_column(
Integer, ForeignKey("systems.id", ondelete="CASCADE")
)
__table_args__ = (
UniqueConstraint("note_id", "system_id", name="uq_record_systems_note_system"),
Index("ix_record_systems_note_id", "note_id"),
Index("ix_record_systems_system_id", "system_id"),
)
+56
View File
@@ -0,0 +1,56 @@
from sqlalchemy import BigInteger, Index, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
SURFACED = "surfaced"
PULLED = "pulled"
class SystemUsageEvent(Base, CreatedAtMixin):
"""One row per time a System's rulings were SHOWN to the agent because its
files were touched, or the System was PULLED in full.
The third of the usage tables, after `note_usage_events` and
`rule_usage_events`, and separate from both for the reason the rule twin
gives: identity at restore. A System id is its own namespace, mapped
through the restore's system map; parked in either sibling's id column it
would come back attached to whatever note or rule took that number.
What it answers: whether an area's rulings, delivered by path rather than
by a ranker, are then read (`get_system`) — and so whether delivery by
path earns its line.
FK-free on `system_id`, `user_id` and `project_id`, like its siblings:
telemetry outlives the row it describes.
"""
__tablename__ = "system_usage_events"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
user_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
system_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
# 'surfaced' | 'pulled'. Plain Text, no CHECK, like the siblings.
event: Mapped[str] = mapped_column(Text, nullable=False)
# Which surface produced it — a convention, not a vocabulary (see the
# note twin). `grep -rn record_system_ src/` is the authoritative list.
source: Mapped[str] = mapped_column(Text, nullable=False)
# The project the READER was in, not the System's own.
project_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
__table_args__ = (
Index("ix_system_usage_system_event", "system_id", "event"),
Index("ix_system_usage_created_at", "created_at"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": iso(self.created_at),
"user_id": self.user_id,
"system_id": self.system_id,
"event": self.event,
"source": self.source,
"project_id": self.project_id,
}
+26
View File
@@ -0,0 +1,26 @@
from sqlalchemy import ForeignKey, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import TimestampMixin, iso
class TaskLog(Base, TimestampMixin):
__tablename__ = "task_logs"
id: Mapped[int] = mapped_column(primary_key=True)
task_id: Mapped[int] = mapped_column(Integer, ForeignKey("notes.id", ondelete="CASCADE"))
user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id", ondelete="CASCADE"))
content: Mapped[str] = mapped_column(Text)
duration_minutes: Mapped[int | None] = mapped_column(Integer, nullable=True)
def to_dict(self) -> dict:
return {
"id": self.id,
"task_id": self.task_id,
"user_id": self.user_id,
"content": self.content,
"duration_minutes": self.duration_minutes,
"created_at": iso(self.created_at),
"updated_at": iso(self.updated_at),
}
+31
View File
@@ -0,0 +1,31 @@
from sqlalchemy import Index, Integer, Text
from sqlalchemy.orm import Mapped, mapped_column
from grimoire.models import Base
from grimoire.models.base import CreatedAtMixin, iso
class User(Base, CreatedAtMixin):
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True)
username: Mapped[str] = mapped_column(Text, unique=True, nullable=False)
email: Mapped[str | None] = mapped_column(Text, nullable=True)
password_hash: Mapped[str | None] = mapped_column(Text, nullable=True)
oauth_sub: Mapped[str | None] = mapped_column(Text, unique=True, nullable=True)
role: Mapped[str] = mapped_column(Text, nullable=False, default="user")
session_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
__table_args__ = (
Index("ix_users_username", "username"),
)
def to_dict(self) -> dict:
return {
"id": self.id,
"username": self.username,
"email": self.email,
"role": self.role,
"created_at": iso(self.created_at),
"has_password": self.password_hash is not None,
}