Compare commits

..
1 Commits
Author SHA1 Message Date
Renovate Bot 5bf55fc488 Add renovate.json 2026-08-04 04:01:43 +00:00
814 changed files with 25566 additions and 123126 deletions
+4 -4
View File
@@ -1,12 +1,12 @@
{
"name": "grimoire-plugin",
"name": "scribe-plugin",
"owner": { "name": "Bryan Van Deusen" },
"description": "Grimoire ships its own Claude Code plugin from this repo, versioned in lockstep with the app + the /api/plugin/context contract.",
"description": "Scribe ships its own Claude Code plugin from this repo, versioned in lockstep with the app + the /api/plugin/context contract.",
"plugins": [
{
"name": "grimoire",
"name": "scribe",
"source": "./plugin",
"description": "Grimoire second brain: MCP tools + session-start push channel + universal process-skills."
"description": "Scribe second brain: MCP tools + session-start push channel + universal process-skills."
}
]
}
+2 -7
View File
@@ -1,9 +1,4 @@
# An install created before the rename from Scribe keeps its database by
# setting these, and PGDATA_VOLUME, to the names it was created with.
POSTGRES_USER=fabled
POSTGRES_PASSWORD=fabled
POSTGRES_DB=grimoire
# PGDATA_VOLUME=grimoire_pgdata
# Grimoire keeps its own SECRET_KEY in the /data volume. Set one here only to
# use a key you manage yourself; it then wins over the kept one.
# SECRET_KEY=
POSTGRES_DB=scribe
SECRET_KEY=dev-secret-change-me
+145 -117
View File
@@ -4,15 +4,27 @@
# Push to main: typecheck + lint + test + build :latest + :<sha>
# Tag v* (release): typecheck + lint + test + build :latest + :<version> + :<sha>
#
# main is the production line, so :latest tracks main's tip. Every push also
# gets an immutable :<sha>, the rollback point. A release (a v* tag created on
# main in the Forgejo UI) adds the dated :<version>.
# Both dev and main are gated AND built. dev pushes move :dev; main pushes move
# :latest — main IS the production line, so :latest tracks main's tip and there
# is no separate :main tag. Every push also gets an immutable :<sha> (the
# rollback point). A v* release tag additionally publishes the dated :<version>;
# since main already moved :latest, the release tag's distinct job is that
# :<version> marker (it refreshes :latest too, harmlessly).
#
# A newer push to the same ref cancels the older run; dev and main never cancel
# each other. Pull requests don't trigger runs: branch pushes already gate.
# Successive pushes to the SAME ref supersede each other (see concurrency
# below), so rapid pushes don't stack identical work; dev and main runs are
# independent refs and never cancel one another.
#
# Forgejo Actions doesn't reliably honour `on.push.branches` as a filter, so
# every job repeats the ref check in its `if:`.
# To cut a release:
# Create a release via the Forgejo UI on main with a v* tag name.
# The tag push triggers this workflow; build job pushes :latest + :<version>.
#
# PRs aren't triggered on purpose — this is a solo dev→main flow, so
# gating on branch push is already enough.
#
# NOTE on the `if:` guards below: Forgejo Actions does not consistently
# honor `on.push.branches` as a filter, so every job repeats the ref check
# explicitly — permitting dev, main, and v* tags, rejecting anything else.
#
# Required secrets (repo → Settings → Secrets → Actions):
# REGISTRY_USER — your Forgejo username
@@ -28,22 +40,33 @@ on:
- "frontend/**"
- "tests/**"
- "pyproject.toml"
# The lock decides what is installed, so a lock-only change must run CI.
# The lock now determines what gets installed, so a lock-only change has
# to trigger a run — otherwise a dependency bump lands untested.
- "uv.lock"
- "alembic/**"
- "alembic.ini"
- "Dockerfile"
# The plugin ships straight from git (via .claude-plugin/marketplace.json),
# not in the image, so a push here is its release. See the `plugin` job.
- "assets/**"
- "fable-mcp/**"
# The plugin ships straight from this repo — installs fetch it via
# .claude-plugin/marketplace.json, NOT from the image. So a push here is
# the release, with no build step in between. Omitting these paths meant
# plugin changes triggered no workflow at all, which is how #2198's three
# broken hooks and then #2209's missing version bump both reached a live
# install. See the `plugin` job below.
- "plugin/**"
- ".claude-plugin/**"
- "scripts/check_plugin.py"
- ".forgejo/workflows/ci.yml"
# Manual run from the Forgejo Actions UI, e.g. to rebuild an image.
# Manual trigger from the Forgejo Actions UI. Useful when an image has
# been built but the deployment didn't pick it up, or when re-running
# against the same source produces different upstream behaviour
# (e.g. a transient HF download flake during the voice-bundle step).
workflow_dispatch: {}
# A newer push cancels the older run on the same branch. Tag runs are never
# cancelled, so a release build can't be cut off mid-flight.
# Cancel older runs on the same branch when a newer push lands. Tag runs
# get their own group implicitly (refs/tags/v1.2.3 ≠ refs/heads/dev) and
# are never cancelled, so a release build can't kill itself mid-flight.
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
@@ -55,7 +78,7 @@ permissions:
env:
REGISTRY: git.fabledsword.com
IMAGE: git.fabledsword.com/bvandeusen/fabledgrimoire
IMAGE: git.fabledsword.com/bvandeusen/fabledscribe
jobs:
typecheck:
@@ -70,7 +93,10 @@ jobs:
- name: Cache npm download cache
uses: actions/cache@v4
# Non-fatal: a cache-backend error just means installing without it.
# Non-fatal: a transient cache-backend hiccup must NOT fail the whole
# typecheck job (it was skipping install + type check and reporting red
# on backend-only pushes — see issue task #828). On cache miss/error the
# job just installs without the cache.
continue-on-error: true
with:
path: ~/.npm
@@ -86,7 +112,8 @@ jobs:
working-directory: frontend
# Guards the one part of this repo that ships to users without a build step.
# scripts/check_plugin.py says what it checks and what it can't.
# See scripts/check_plugin.py for what it checks and, as importantly, what it
# can't check yet (shellcheck and jq are absent from ci-python).
plugin:
name: Plugin hooks
if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
@@ -94,18 +121,26 @@ jobs:
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
steps:
# Keep this a bare `uses:`: with a `with:` block the action fails to
# extract on act_runner ("Cannot find module .../dist/index.js"). The
# version check diffs two trees, so a depth-1 fetch of main is enough.
# Bare `uses:`, no `with:` block. Adding one made this action fail to
# extract on the act_runner ("Cannot find module .../dist/index.js") while
# every bare checkout in the same run succeeded — see run 3027. Nothing
# here needs `fetch-depth: 0` anyway: the version check diffs two trees,
# and a tree diff needs both trees, not a common ancestor. A depth-1 fetch
# of main's tip is enough, and cheaper.
- uses: actions/checkout@v6
# Per-job rather than in the image: Grimoire is shellcheck's only consumer.
# Install nothing else here; the hooks use only POSIX tools, and the
# smoke test runs on a bare image to prove it.
# Per-job, not in the image, per CI-runner's docs/process.md: "If only one
# project needs the dep, prefer that project installing it per-job in
# their workflow — at least until a second consumer arrives." Scribe is
# the only consumer today. Promotion into ci-python is filed as an issue
# on CI-runner rather than assumed here.
#
# jq is not optional for the smoke test: every hook exits at line 1
# without it, so the check would pass while exercising nothing.
- name: Install shell tooling
run: |
apt-get update -qq
apt-get install -y -qq --no-install-recommends shellcheck
apt-get install -y -qq --no-install-recommends jq shellcheck
# On main the comparison would be against itself, so only the syntax and
# pattern checks mean anything there.
@@ -127,22 +162,22 @@ jobs:
steps:
- uses: actions/checkout@v6
# ruff is pre-installed in the ci-python image.
# ruff is pre-installed in the ci-python image — no install
# step needed, lint runs in ~2s.
- name: Lint
run: ruff check src/ scripts/
# Checks the frontend's CSS against the committed, generated theme.css.
# Fails only on an unresolvable var() reference; hard-coded literals are
# reported, not gated.
# Design tokens: does the frontend's CSS agree with the stylesheet the
# design system generates? Fails only on an unresolvable var() reference —
# that count is at zero, so this is a ratchet rather than a backlog. The
# literal findings are printed, not gated; hundreds exist and a
# permanently-red job is one nobody reads.
#
# Stdlib only, no install, no network: the source of truth is theme.css,
# which is generated from the design system and committed.
- name: Design token check
run: python3 scripts/check_design_tokens.py --report-literals
# Reports elements whose classes have modifier rules but no base rule,
# which vue-tsc can't see. Not gated: a bare wrapper is legitimate, so
# the signal is the count growing.
- name: Dangling style check
run: python3 scripts/check_dangling_styles.py
test:
name: Python tests
if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
@@ -156,87 +191,87 @@ jobs:
uses: actions/cache@v4
with:
path: ~/.cache/uv
# Keyed on the lock, which decides the installed set.
# Keyed on the LOCK, not pyproject: the lock is what determines the
# installed set now, and a pyproject edit that doesn't change
# resolution shouldn't throw the cache away.
key: uv-${{ hashFiles('uv.lock') }}
restore-keys: uv-
# Installs exactly what uv.lock pins, so green means these exact versions
# passed. `--locked` fails if the lock is stale against pyproject, so a
# dependency change needs a deliberate `uv lock`.
# Installs exactly what uv.lock pins, and resolves nothing itself.
#
# This replaced `uv pip install -e ".[dev]"`, which resolved from the
# pyproject constraints and ignored the lock entirely. Every dependency
# floated: on 2026-07-28 mcp 2.0.0 shipped mid-session and turned `main`
# red with no repo change (issue #2194). Green CI has to mean "these exact
# versions passed", or it isn't evidence of anything.
#
# `--locked` also FAILS when uv.lock is stale against pyproject, so a
# dependency edit has to go through a deliberate `uv lock` — it can't
# arrive on its own. That check earned its place immediately: it caught
# that the lock had been missing `pgvector` entirely (added to pyproject,
# never re-locked), which the old install path had been silently papering
# over by resolving from pyproject instead.
- name: Install locked dependencies
env:
UV_PROJECT_ENVIRONMENT: /opt/venv
run: uv sync --locked --extra dev
# The hook execution tests run on a bare image (no jq), like a user's
# machine might be.
- name: Run tests
# Integration tests (real Postgres) run in the `integration` job below.
run: /opt/venv/bin/python -m pytest tests/ -q -m "not integration"
# Real Postgres, built by the real migrations: covers the async SQLAlchemy
# paths unit stubs can't reach.
# Real-Postgres lane (family rule 6). Exercises the async SQLAlchemy connection
# path the unit stubs can't reach — the un-awaited execution_options regression
# that made every VACUUM report 0/6 lived here. Like `test`, it runs for
# visibility and does NOT gate the build.
#
# Service hostnames aren't routable on this runner, so the step finds the
# Postgres container's bridge IP. The job has no `name:` and a separator-free
# key because act_runner names service containers after it.
# Job key stays separator-free ("integration"): act_runner derives the service-
# container name from the (truncated) job display name and the discovery step
# filters `docker ps` by it. Service hostnames aren't routable on this runner,
# so the step resolves the Postgres container's bridge IP. No `name:` on purpose.
integration:
if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
env:
# Read at import time. DATABASE_URL is built from the discovered IP below.
# Config + the module engine read these at import time. DATABASE_URL itself
# is built from the discovered service IP in the run step.
SECRET_KEY: ci_integration_placeholder
services:
postgres:
# Same image as the shipped compose files: Postgres 17 with pgvector.
# pgvector image so `alembic upgrade head` can run migration 0067
# (CREATE EXTENSION vector). PG17 — matches the prod/quickstart image.
image: pgvector/pgvector:pg17
env:
POSTGRES_USER: grimoire
POSTGRES_USER: scribe
POSTGRES_PASSWORD: ci_integration
POSTGRES_DB: grimoire_test
POSTGRES_DB: scribe_test
options: >-
--health-cmd "pg_isready -U grimoire"
--health-cmd "pg_isready -U scribe"
--health-interval 10s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@v6
# Same locked install as the unit lane, so both test the same versions.
# Same locked install as the unit lane — the two must agree on versions,
# or "unit green, integration red" stops being a signal about the code.
- name: Install locked dependencies
env:
UV_PROJECT_ENVIRONMENT: /opt/venv
run: uv sync --locked --extra dev
# Prints two facts about the runner that the steps below depend on.
# Every command is guarded so the diagnostic can't fail the lane.
- name: Runner facts (shell, service hostnames)
run: |
echo "--- which shell runs a run: step ---"
readlink -f /bin/sh || echo "/bin/sh: not a symlink"
ps -p $$ -o comm= || true
echo "--- is a service reachable by its hostname? ---"
getent hosts postgres \
|| echo "no — 'postgres' does not resolve; the bridge-IP lookup is still required"
- name: Integration suite (resolve service IP, migrate, test)
run: |
set -eux
echo "=== container landscape (diagnostic for the name filter) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
# Only this job's services: the docker daemon is shared, so a job-name
# filter can match another repo's Postgres. act names a task's containers
# GITEA-ACTIONS-TASK-<n>-...; read <n> from our own container (its id is in
# the /etc/hostname bind mount) and require exactly one match.
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
SELF=${SELF:-$(hostname)}
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
test -n "$TASK"
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg17" -q)
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg17" -q | head -n1)
test -n "$PG"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
test -n "$PG_IP"
export DATABASE_URL="postgresql+asyncpg://grimoire:ci_integration@${PG_IP}:5432/grimoire_test"
# Wait for Postgres. The run: shell is dash (no bash /dev/tcp), so use Python.
export DATABASE_URL="postgresql+asyncpg://scribe:ci_integration@${PG_IP}:5432/scribe_test"
# Wait for Postgres to accept connections (busybox sh — the runner
# default — has no bash /dev/tcp, so use Python).
/opt/venv/bin/python - "$PG_IP" <<'PY'
import socket, sys, time
for _ in range(30):
@@ -255,12 +290,15 @@ jobs:
build:
name: Build & push image
# `plugin` is not in needs: the plugin ships from git, not in the image, so
# holding the image back wouldn't stop a bad hook. A red `plugin` job still
# turns the run red. `integration` is: the image carries the migrations and
# queries that lane tests.
needs: [typecheck, lint, test, integration]
# dev → :dev, main → :latest, tag → :latest + :<version>; all get :<sha>.
# `plugin` is deliberately NOT in needs. The plugin isn't in the image —
# installs fetch it from git — so gating the server image on a hook lint
# would couple two things that don't ship together, and blocking the build
# wouldn't un-publish a bad hook anyway: the push already did that. A failed
# `plugin` job still turns the whole run red, which is the signal that
# matters.
needs: [typecheck, lint, test]
# Build on dev, main, and v* tag pushes. dev → :dev, main → :latest,
# tag → :latest + :<version>; every build also gets an immutable :<sha>.
if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
runs-on: python-ci
container:
@@ -270,55 +308,47 @@ jobs:
packages: write
steps:
- uses: actions/checkout@v6
with:
# Full history for the version derivation. Today it needs only HEAD's
# commit time, but a shallow clone fails silently (a too-low value) if
# the derivation ever reads further back.
fetch-depth: 0
- name: Generate image tags and version
id: tags
# POSIX `case`, not bash `[[ ]]`: the run: shell is dash, where `[[`
# fails silently and only the :<sha> tag would be pushed.
# POSIX `case` instead of bash `[[ ]]` because act_runner invokes
# `sh -e` (dash on the ci-python:3.14 image, which has no bash on
# the default PATH for /bin/sh). Previous `[[ ]]` form failed
# silently — only the SHA tag got appended, so :dev / :latest
# never updated in the registry and the deployed stack kept
# pulling stale images. Verified via `[[: not found` lines in
# the runner log on commit 2a374d9.
run: |
TAGS="${{ env.IMAGE }}:${{ github.sha }}"
# Three values kept separate; the Dockerfile describes each.
# 1. Ordering key: build time in minutes since 2020-01-01, so it only
# goes up. Commit time would go down when an older commit is
# rebuilt; a commit count isn't monotonic across branches.
BUILD_KEY=$(( ( $(date -u +%s) - 1577836800 ) / 60 ))
# 2. Name: commit time, so the same source reads the same on every lane.
COMMIT_TS=$(git log --format=%ct -1 HEAD)
BUILD_NAME=$(date -u -d "@$COMMIT_TS" +%Y.%m.%d.%H%M)
# 3. Channel: its own value, never part of the name.
CHANNEL="dev"
BUILD_VERSION="dev"
case "${{ github.ref }}" in
refs/heads/dev)
TAGS="$TAGS,${{ env.IMAGE }}:dev"
;;
refs/heads/main)
# main is the production line, so it publishes :latest.
# main IS the production line: publish :latest (plus the :<sha>
# set above). No separate :main tag.
TAGS="$TAGS,${{ env.IMAGE }}:latest"
CHANNEL="stable"
BUILD_VERSION="main"
;;
refs/tags/*)
TAGS="$TAGS,${{ env.IMAGE }}:latest,${{ env.IMAGE }}:${{ github.ref_name }}"
CHANNEL="stable"
BUILD_VERSION="${{ github.ref_name }}"
;;
esac
echo "value=$TAGS" >> $GITHUB_OUTPUT
echo "build_name=$BUILD_NAME" >> $GITHUB_OUTPUT
echo "build_key=$BUILD_KEY" >> $GITHUB_OUTPUT
echo "channel=$CHANNEL" >> $GITHUB_OUTPUT
echo "build_version=$BUILD_VERSION" >> $GITHUB_OUTPUT
# No disk-space prune step: the runner's Docker daemon is shared, and
# `docker system prune -af` here deleted images other repos' jobs had
# just pulled and were about to start (CI-runner #16). The runner host
# prunes nightly instead.
- name: Free disk space
# Self-hosted runner housekeeping. Two-step cleanup:
# 1. Prune dangling containers/images globally (stops the runner
# from accumulating cruft from past failed builds).
# 2. Trim the BuildKit layer cache to a 5GB ceiling so the pip
# mount cache survives but old intermediate layers don't
# accumulate indefinitely.
run: |
docker system prune -af || true
docker builder prune --keep-storage 5g -f || true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
@@ -337,13 +367,11 @@ jobs:
push: true
provenance: false
tags: ${{ steps.tags.outputs.value }}
# The commit lets the image's self-report be checked against :<sha>.
build-args: |
BUILD_VERSION=${{ steps.tags.outputs.build_name }}
BUILD_KEY=${{ steps.tags.outputs.build_key }}
BUILD_CHANNEL=${{ steps.tags.outputs.channel }}
BUILD_COMMIT=${{ github.sha }}
# Layer cache kept in the registry, so a build starts warm even after
# the runner prunes. `mode=max` caches intermediate layers too.
build-args: BUILD_VERSION=${{ steps.tags.outputs.build_version }}
# Registry-backed layer cache. Pull from :cache to prime
# BuildKit, push updated layers back to :cache so the next
# build starts warm even if the runner's local cache was
# pruned. `mode=max` exports all intermediate layers, not
# just the final image, which is what gives the ~80% speedup.
cache-from: type=registry,ref=${{ env.IMAGE }}:cache
cache-to: type=registry,ref=${{ env.IMAGE }}:cache,mode=max,ignore-error=true
+9 -23
View File
@@ -12,8 +12,11 @@ RUN npm run build
FROM python:3.14-slim AS runtime
WORKDIR /app
# Installed from uv.lock, exactly like CI, so the image runs the dependency
# versions CI tested.
# Installed from uv.lock, exactly like CI (issue #2194). This used to be
# `COPY pyproject.toml .` + `pip install .`, which never even copied the lock:
# the shipped image resolved its own dependency set, so CI could be green on one
# set of versions while the published image ran another. On 2026-07-28 that
# class of drift turned `main` red when mcp 2.0.0 shipped mid-session.
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-cache-dir uv
@@ -31,34 +34,17 @@ RUN --mount=type=cache,target=/root/.cache/uv \
# alembic and hypercorn in CMD have to be found there.
ENV PATH="/app/.venv/bin:$PATH"
COPY --from=build-frontend /build/dist/ src/grimoire/static/
COPY --from=build-frontend /build/dist/ src/scribe/static/
COPY alembic.ini .
COPY alembic/ alembic/
# Ensure Python finds the source tree (where static files live) before site-packages
ENV PYTHONPATH=/app/src
# Three values kept separate, plus the commit:
# BUILD_VERSION is the name (YYYY.MM.DD.HHMM from commit time), the same on
# every lane for the same source.
# BUILD_KEY is the ordering key (minutes since 2020-01-01 from build time), the
# only value compared to decide which build is newer.
# BUILD_CHANNEL is its own field, never part of the name.
# BUILD_COMMIT lets the image's self-report be checked against its :<sha> tag.
# A local build leaves the key and channel empty: it has no place in an update
# order, and /api/version omits them.
# Version is injected at build time via --build-arg BUILD_VERSION=YY.MM.DD.N
# Falls back to "dev" for local / untagged builds
ARG BUILD_VERSION=dev
ARG BUILD_KEY=
ARG BUILD_CHANNEL=
ARG BUILD_COMMIT=
ENV APP_VERSION=$BUILD_VERSION
ENV APP_BUILD_KEY=$BUILD_KEY
ENV APP_CHANNEL=$BUILD_CHANNEL
ENV APP_COMMIT=$BUILD_COMMIT
# The instance's own SECRET_KEY, the model cache and diagnostics live here.
# Mount a volume on it so they outlive the container.
RUN mkdir -p /data
EXPOSE 5000
CMD ["sh", "-c", "alembic upgrade head && hypercorn 'grimoire.app:create_app()' --bind 0.0.0.0:5000 --keep-alive 600"]
CMD ["sh", "-c", "alembic upgrade head && hypercorn 'scribe.app:create_app()' --bind 0.0.0.0:5000 --keep-alive 600"]
-661
View File
@@ -1,661 +0,0 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<https://www.gnu.org/licenses/>.
+7 -15
View File
@@ -1,26 +1,26 @@
.PHONY: build up down logs health migrate lint typecheck test fmt mint-plugin
.PHONY: build up down logs health migrate lint typecheck test fmt
# --- Docker ---
build:
docker compose -f docker-compose.dev.yml build
docker compose build
up:
docker compose -f docker-compose.dev.yml up -d
docker compose up -d
down:
docker compose -f docker-compose.dev.yml down
docker compose down
logs:
docker compose -f docker-compose.dev.yml logs -f
docker compose logs -f
health:
curl -s http://localhost:5000/api/health
migrate:
docker compose -f docker-compose.dev.yml exec app alembic upgrade head
docker compose exec app alembic upgrade head
# --- Local dev (requires: uv sync --extra dev and cd frontend && npm ci) ---
# --- Local dev (requires: pip install -e ".[dev]" and cd frontend && npm ci) ---
lint:
ruff check src/
@@ -36,11 +36,3 @@ test:
# Run all checks in one shot (mirrors what CI does)
check: lint typecheck test
# --- Plugin ---
# Run after changing plugin/ or .claude-plugin/, before committing: the plugin
# ships from git with no build step, so its version is minted here. CI fails if
# the version wasn't bumped.
mint-plugin:
python3 scripts/mint_plugin_version.py
+27 -138
View File
@@ -1,151 +1,40 @@
<p align="center">
<img src="frontend/public/favicon.svg" width="96" alt="">
</p>
# Fabled Scribe
<h1 align="center">Fabled Grimoire</h1>
<p align="center">A self-hosted system of record for the work you do with Claude Code.</p>
Claude records what it learned, decided and built in Grimoire, and that record reaches the next session, so a solution is recalled instead of reinvented. Notes, tasks, plans, rules, lessons and reusable code live in one place. Claude reaches them through a built-in MCP endpoint and the bundled Claude Code plugin, and people read and edit the same records in the web UI. There is no language model inside Grimoire: Claude is the only assistant.
## Deploy
You need Docker with Compose. Save this as `docker-compose.yml` (it is the [`docker-compose.yml`](docker-compose.yml) in this repository), then run `docker compose up -d` beside it:
```yaml
# Fabled Grimoire: the app and its database.
# docker compose up -d
# Then open http://localhost:5000. The first account registered becomes admin.
#
# Keep both volumes. app_data holds the instance's own SECRET_KEY, which seals
# saved credentials; pgdata holds everything else.
services:
app:
image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest
ports:
- "5000:5000"
environment:
DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}"
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
# Behind an HTTPS reverse proxy:
# BASE_URL: "https://grimoire.example.com"
# SECURE_COOKIES: "true"
# TRUST_PROXY_HEADERS: "true"
volumes:
- app_data:/data
depends_on:
- db
restart: unless-stopped
db:
# Postgres 17 with the vector extension Grimoire needs.
image: pgvector/pgvector:pg17
stop_grace_period: 120s
volumes:
- pgdata:/var/lib/postgresql/data
environment:
POSTGRES_USER: ${POSTGRES_USER:-grimoire}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire}
POSTGRES_DB: ${POSTGRES_DB:-grimoire}
restart: unless-stopped
volumes:
app_data:
pgdata:
```
Open `http://localhost:5000`. The first account you register becomes the admin.
Nothing has to be set first. On its first start Grimoire makes its own `SECRET_KEY` and keeps it in the `app_data` volume, so restarts, reboots and updates all use the same key.
To connect Claude, create an API key under **Settings → MCP Access**, then in Claude Code:
```
/plugin marketplace add https://git.fabledsword.com/bvandeusen/FabledGrimoire.git
/plugin install grimoire@grimoire-plugin
```
The plugin asks for your Grimoire URL and the API key. Settings → MCP Access shows the exact commands for your instance.
## Configuration
Everything is optional. Set these in the environment or in a `.env` file beside the compose file:
| Variable | When | What it does |
|----------|------|--------------|
| `BASE_URL` | Behind a proxy | The public URL, used in email links and SSO redirects |
| `SECURE_COOKIES` | Over HTTPS | `true` when Grimoire is served over HTTPS |
| `TRUST_PROXY_HEADERS` | Behind a proxy | `true` to read the client address from the proxy's headers |
| `POSTGRES_PASSWORD` | Before the first start | The database password (defaults to `grimoire`; the database is reachable only inside the compose network). Postgres takes it once, when the volume is created |
| `SECRET_KEY` | Rarely | A key you manage yourself, instead of the one Grimoire keeps. It wins over the kept key; remove it to go back |
Everything else (registration, email, SSO, git forge connections, retrieval tuning, backups) is in **Settings**, or listed in [Configuration](docs/configuration.md).
## Setup pitfalls
- **Keep both volumes.** `pgdata` holds everything Grimoire stores. `app_data` holds the instance's key, and forge tokens and the backup bucket's secret are sealed under it. If the key is lost, Settings asks you to enter those credentials again; nothing else is lost.
- **The embedding model downloads on first use**, into the `app_data` volume. Without internet access, semantic search falls back to keyword search until the model is in place.
- **Behind a reverse proxy, turn off response buffering** for `/mcp`, because MCP responses stream. [Configuration](docs/configuration.md#behind-a-reverse-proxy) has an Nginx example.
## Updating and backups
```bash
docker compose pull
docker compose up -d
```
Database migrations run when the app starts, and the key stays in its volume. Claude Code updates the plugin from its marketplace; run `/reload-plugins` after an update.
**Settings → Data → Scheduled backups** writes a daily backup of the database to a folder or an S3-compatible bucket, and `python -m grimoire.restore` brings a fresh instance back from one. See [Backups](docs/backups.md).
A self-hosted work system-of-record for software projects, built to be driven by Claude Code. Notes, tasks, issues, projects, milestones, rules, and stored processes — reachable from Claude via a built-in MCP endpoint and a bundled Claude Code plugin, with a clean web UI for humans. No in-app LLM; Claude is the sole assistant.
## Features
**Working with Claude**
- Claude enters a project and gets a short primer: its goal, current plans, open tasks and areas.
- Your rules reach Claude when its work matches them, or at a moment of work such as finishing a task or pushing code.
- Before Claude writes a file or runs a command, the plugin shows it the rules, lessons and reusable code that may apply.
- Stored procedures, such as a drift audit, become Claude Code skills.
Notes and tasks with a Markdown editor, sub-tasks, milestones, issues, and kanban project workspaces. Stored processes, an engineering rulebook system, and semantic search with proactive knowledge-injection into Claude's context. A knowledge graph, per-user/group sharing, and a built-in MCP server (`/mcp`) plus a bundled Claude Code plugin so Claude can record and recall your work directly.
**Recording the work**
- Notes in a Markdown editor, with wikilinks, tags and version history.
- Tasks, issues and time-boxed investigations, with work logs that the next session reads first.
- Plans as milestones: a design plus ordered steps.
- Lessons learned the hard way, linked to the rules they illustrate.
- Duplicate checks, so a new record that matches an existing one updates it instead.
## Quick Start
**Reusing code**
- Snippets of reusable code, each with when to reach for it and where it lives in the repo.
- With a git forge connected, Grimoire tracks every function and style in your repos and shows which ones are still unaccounted for.
- Ideas shared across projects on the same platform, with each project's answer recorded.
- Design systems with tokens, inheritance and a generated stylesheet.
**Prerequisites:** Docker and Docker Compose. No GPU or local model needed — Claude is the sole assistant, reached over MCP.
**For people**
- A web UI to browse, search and edit everything, plus a graph of how notes connect.
- Sharing with users and groups, as viewer, editor or admin.
- Email and password sign-in, optional SSO through OIDC, and invite links.
- Personal export to Markdown or JSON, and admin backup and restore.
The full list is in [Features](docs/features.md).
## Troubleshooting
| What you see | Why | Fix |
|--------------|-----|-----|
| `SECRET_KEY is set to the public placeholder` at startup | `SECURE_COOKIES=true` while `SECRET_KEY` is `dev-secret-change-me` | Remove `SECRET_KEY`, or set a real one |
| Settings asks for forge tokens or the backup secret again | The instance's key changed: the `app_data` volume was replaced, or `SECRET_KEY` was set or removed | Enter them again, or restore the previous key |
| `Can't locate revision identified by '…'` at startup | An older image is running against a database a newer one has already migrated | Run the newer image; migrations only move forward |
| The plugin's tools are missing in Claude Code | The plugin isn't installed, or Claude Code hasn't reloaded it | `/plugin` to install or update it, then `/reload-plugins` |
| Search finds only exact words | The embedding model hasn't downloaded yet | Give the app internet access once, or wait for the first download to finish |
## Development
Download [`docker-compose.quickstart.yml`](docker-compose.quickstart.yml) from this repo, then:
```bash
docker compose -f docker-compose.dev.yml up --build
# Optional but recommended — set a secret key
export SECRET_KEY=your-random-secret-here
docker compose -f docker-compose.quickstart.yml up -d
```
[Development](docs/development.md) covers checks, migrations, the plugin and CI. [Architecture](docs/architecture.md) describes how it is built, and the [API reference](docs/api-reference.md) lists every REST endpoint.
Open `http://localhost:5000`. The first user to register becomes admin. To connect Claude, create an API key under **Settings → API Keys** and install the Claude Code plugin — see [API Keys & MCP](docs/api-keys-and-mcp.md).
## Licence
> **Development:** To build from source, see [Development](docs/development.md).
Fabled Grimoire is free software under the [GNU Affero General Public License v3.0 only](LICENSE) (`AGPL-3.0-only`). If you run a modified version for other people over a network, the licence requires you to offer them its source.
## Documentation
| Doc | Contents |
|-----|----------|
| [Architecture](docs/architecture.md) | Stack, design decisions, data models, key services |
| [Configuration](docs/configuration.md) | Environment variables, Docker Compose, production setup, security |
| [Features](docs/features.md) | Detailed feature breakdown and keyboard shortcuts |
| [Development](docs/development.md) | Dev workflow, CI/CD, migrations, release process |
| [API Keys & MCP](docs/api-keys-and-mcp.md) | API key management and Fable MCP install guide |
| [SSO / OAuth](docs/sso-oauth.md) | OIDC setup for Authentik, Keycloak, and other providers |
| [API Reference](docs/api-reference.md) | All REST API endpoints |
## License
This project is privately maintained.
+2 -4
View File
@@ -4,9 +4,8 @@ from logging.config import fileConfig
from alembic import context
from sqlalchemy.ext.asyncio import create_async_engine
from grimoire.config import Config
from grimoire.models import Base
from grimoire.services.db_ready import wait_for_database
from scribe.config import Config
from scribe.models import Base
config = context.config
if config.config_file_name is not None:
@@ -33,7 +32,6 @@ def do_run_migrations(connection):
async def run_async_migrations() -> None:
await wait_for_database(Config.DATABASE_URL)
engine = create_async_engine(Config.DATABASE_URL)
async with engine.connect() as connection:
await connection.run_sync(do_run_migrations)
@@ -1,11 +1,13 @@
"""replace events.end_dt with duration_minutes
"""replace events.end_dt with duration_minutes (Fable #160)
Revision ID: 0043
Revises: 0042
Create Date: 2026-04-29
Structural fix for the "end before start" bug class: an event whose end_dt
precedes its start_dt disappears from upcoming-list filters. Storing duration instead of end_dt makes the invalid state
Structural fix for the "end before start" bug class observed on prod
2026-04-29: an event landed with end_dt 32 days before start_dt due
to a tool-call mishap, then disappeared from upcoming-list filters.
Storing duration instead of end_dt makes the invalid state
inexpressible at the schema level (duration_minutes >= 0).
Backfill rules:
@@ -5,7 +5,7 @@ Revises: 0043
Create Date: 2026-05-13
Adds two columns to the ``notes`` table to support the task-as-durable-record
design:
design (spec 2026-05-13):
- ``description``: user-stated goal / initial context. Meaningful when
``is_task=true``; left NULL on knowledge notes.
@@ -4,7 +4,7 @@ Revision ID: 0048
Revises: 0047
Create Date: 2026-05-22
Part of the conversation+curator architecture. The
Phase 2 of the conversation+curator architecture (Fable #172). The
scheduler runs every 15 minutes and processes any journal conversation
with messages newer than its `last_curator_run_at` timestamp. NULL
means "never run; process all of today on first sweep."
@@ -4,7 +4,7 @@ Revision ID: 0049
Revises: 0048
Create Date: 2026-05-22
Part of the conversation+curator architecture. The
Phase 3 of the conversation+curator architecture (Fable #172). The
curator's final summary line (≤ 240 chars) is persisted here so the
NEXT chat turn can include it in the system prompt. This is the
feedback loop that closes the architecture — without it, the chat
@@ -4,11 +4,13 @@ Revision ID: 0053
Revises: 0052
Create Date: 2026-05-27
Part of the MCP-first pivot. The Python models for these tables are gone;
this migration drops the orphan
Phase 9 of the MCP-first pivot. The Python models for these tables were
deleted in Phase 8 (commit 91bafb6); this migration drops the orphan
SQL tables and cleans up dead per-user setting rows.
Hard-cutover: existing rows in these tables are discarded, deliberately.
Hard-cutover: existing rows in these tables are discarded. The
accompanying spec at docs/superpowers/specs/2026-05-26-mcp-first-pivot-design.md
explicitly accepted this loss.
"""
from alembic import op
+1 -1
View File
@@ -4,7 +4,7 @@ Revision ID: 0054
Revises: 0053
Create Date: 2026-05-27
The image cache was wired into the LLM image-search tool, which is gone.
The image cache was wired into the LLM image-search tool (removed in Phase 8).
With no producer or consumer left, the table is dropped here.
"""
from alembic import op
+2 -2
View File
@@ -6,8 +6,8 @@ Create Date: 2026-05-27
Adds the Scribe Rulebook hierarchy as a fourth top-level entity, sibling
to Project. Rules carry a structural (statement, why, how_to_apply)
triple. Projects subscribe to Rulebooks (many-to-many; migration 0101
later drops subscriptions).
triple. Projects subscribe to Rulebooks (many-to-many). See the design
doc at docs/superpowers/specs/2026-05-27-scribe-rulebook-design.md.
"""
from alembic import op
import sqlalchemy as sa
+1 -1
View File
@@ -4,7 +4,7 @@ Revision ID: 0066
Revises: 0065
Create Date: 2026-06-14
The milestone becomes the plan container: its `body` holds
T3 of plan #819. The milestone becomes the plan container: its `body` holds
the design/intent/purpose (markdown), `description` stays the one-liner, and
individual steps live as first-class child tasks (milestone_id) instead of
checkboxes crammed into a kind=plan task body. start_planning is reworked to
+3 -2
View File
@@ -4,7 +4,8 @@ Revision ID: 0074
Revises: 0073
Create Date: 2026-07-31
Prose doesn't live as one-offs; it lives centrally on the system. Two fields rather than one per category:
The operator's shape for #254: prose doesn't live as one-offs, it lives centrally
on the system. Two fields rather than one per category:
design_systems.guidance the narrative a token table cannot hold — aesthetic,
voice and tone, what is deliberately out of scope.
@@ -16,7 +17,7 @@ Prose doesn't live as one-offs; it lives centrally on the system. Two fields rat
Free-form rather than a column per category on purpose. A schema with `voice`,
`aesthetic` and `scope` columns would bake one rulebook's table of contents into
every install, and the next install's design system would have three
every install (rule #115), and the next install's design system would have three
empty columns and nowhere to put what it actually cares about.
Both nullable: a design system with no prose at all is complete, not a draft.
@@ -1,47 +0,0 @@
"""retire the two settings that designated a design source for the app itself
Revision ID: 0075
Revises: 0074
Create Date: 2026-08-03
Two keys, retired for the same reason a week apart, so they go in one change
rather than one migration each:
design_rulebook_id which rulebook described how this app should look
ui_design_system_id which design system this app's own UI was built from
Both named a design source for THE RUNNING INSTALL. The design surface is for
the projects an install tracks, and a project already carries its own pointer
(`projects.design_system_id`) — so an install-wide designation had nothing left
to mean. `ui_design_system_id` was introduced by this same migration's first
draft and never reached a deployed database; it is listed here rather than
undone by an 0076 that would reverse a change nobody ran.
Deleting settings rows by key is safe in a way dropping a column is not — the
table is free-form key/value, so an install that never designated one simply has
no row to delete.
Downgrade cannot restore what it never recorded, so it is a no-op rather than a
lie: the pointer lives on the project now, and always did for anyone who set it
there.
"""
from alembic import op
import sqlalchemy as sa
revision = "0075"
down_revision = "0074"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.execute(
sa.text(
"DELETE FROM settings "
"WHERE key IN ('design_rulebook_id', 'ui_design_system_id')"
)
)
def downgrade() -> None:
pass
-115
View File
@@ -1,115 +0,0 @@
"""note_supersessions; drop the never-written notes.consolidated_at
Revision ID: 0076
Revises: 0075
Create Date: 2026-08-07
Structure only — nothing reads or writes the new
table yet, and nothing behaves differently after this runs.
## What the table is for
Old records outrank newer ones on the same subject, because a similarity score
cannot tell time. A note that accurately described how something worked in June
is still accurate ABOUT June; it is just no longer the answer. Nothing recorded
that, so nothing could act on it.
The claim points FORWARD — the newer record names what it overtakes — because
the older one cannot know it has been overtaken. Many-to-many and partial: a
note may supersede parts of several others and be overtaken piecemeal by
several later ones, which is why this is a table rather than a column. Both
directions are queried: `superseded_id` answers "has this been overtaken?" at
ranking time, `superseder_id` answers "what does this replace?" in a record
view. An array column could serve one and not the other.
CASCADE on both sides is safe because trashing is not a delete: `trash_svc`
stamps `deleted_at`, so a trashed note keeps its claims and `restore` brings
them back. The cascade fires only on `purge_trash`, where the row genuinely
goes — and a claim about a row that no longer exists is not actionable.
## What is being dropped, and why now
`notes.consolidated_at` was written by NOTHING — no service, no route, no tool
— while being serialised into every note and task payload as `null`. It cost a
column, a line in every response, and worse: it IMPLIED a capability. A reader
reasonably concludes notes can be consolidated and this records when.
That reading was reasonable precisely because merge/unmerge exists for snippets
and not for notes, so the column looked like the notes-side half of that
feature, modelled and abandoned.
It is dropped rather than repurposed for supersession, and the distinction is
the point: consolidation folds several records into one survivor and
destroys the originals. Merging two snippets is lossless — one helper, several
call sites. Folding two dev-logs means writing a summary and losing what each
actually said. Supersession is the opposite act: both records survive, and the
older one is merely ranked behind. Smuggling one in under a column named for
the other would have buried that difference in schema.
## Downgrade
Re-adds `consolidated_at` nullable, which is how it lived — so downgrade
restores the shape, not the (nonexistent) data. Drops the table; any recorded
supersession claims are lost, which costs ranking its input and nothing else,
since no note's own content depends on them.
"""
import sqlalchemy as sa
from alembic import op
revision = "0076"
down_revision = "0075"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"note_supersessions",
sa.Column("id", sa.Integer, primary_key=True),
sa.Column(
"superseder_id",
sa.Integer,
sa.ForeignKey("notes.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"superseded_id",
sa.Integer,
sa.ForeignKey("notes.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.UniqueConstraint(
"superseder_id", "superseded_id", name="uq_note_supersessions_pair"
),
# Declaring that a note supersedes ITSELF is meaningless, and under flat
# demotion it would demote a record on its own authority. Refused in the
# service too, with a message — this is the backstop that holds when
# something writes rows directly.
sa.CheckConstraint(
"superseder_id <> superseded_id", name="ck_note_supersessions_not_self"
),
)
op.create_index(
"ix_note_supersessions_superseder", "note_supersessions", ["superseder_id"]
)
op.create_index(
"ix_note_supersessions_superseded", "note_supersessions", ["superseded_id"]
)
op.drop_column("notes", "consolidated_at")
def downgrade() -> None:
op.add_column(
"notes",
sa.Column("consolidated_at", sa.DateTime(timezone=True), nullable=True),
)
op.drop_index("ix_note_supersessions_superseded", table_name="note_supersessions")
op.drop_index("ix_note_supersessions_superseder", table_name="note_supersessions")
op.drop_table("note_supersessions")
@@ -1,54 +0,0 @@
"""Chunked embeddings: one note_embeddings row per chunk
Revision ID: 0077
Revises: 0076
Create Date: 2026-08-09
The embedding model reads at most 512 tokens and fastembed truncates the rest
silently, so the old one-row-per-note shape permanently lost everything past
~400 words of a record. A note now stores one row per chunk of
`embeddings.chunk_document`: PK (note_id, chunk_index), plus the chunk's text
(inspectability + future "matched section" surfacing) and the chunker version
that produced it (so later shape changes re-embed by version comparison
instead of repeating this wipe).
Embeddings are DERIVED data (0067 precedent): rows are cleared here and the
startup backfill regenerates the whole corpus at the new shape on next boot.
The HNSW index is untouched — it indexes chunk rows exactly as it indexed
note rows.
"""
from alembic import op
revision = "0077"
down_revision = "0076"
branch_labels = None
depends_on = None
def upgrade() -> None:
# Derived data — the version-aware startup backfill re-embeds everything
# at the chunked shape. Old whole-document rows would be indistinguishable
# from properly-chunked single-chunk notes, so they cannot be carried over.
op.execute("DELETE FROM note_embeddings")
# Empty table, so NOT NULL columns need no defaults and the PK swap is
# instant.
op.execute("ALTER TABLE note_embeddings ADD COLUMN chunk_index integer NOT NULL")
op.execute("ALTER TABLE note_embeddings ADD COLUMN chunk_text text NOT NULL")
op.execute("ALTER TABLE note_embeddings ADD COLUMN chunker_version integer NOT NULL")
op.execute("ALTER TABLE note_embeddings DROP CONSTRAINT note_embeddings_pkey")
op.execute(
"ALTER TABLE note_embeddings ADD PRIMARY KEY (note_id, chunk_index)"
)
def downgrade() -> None:
# Same reasoning in reverse: chunk rows make no sense to a whole-document
# reader, so clear and let the old backfill regenerate.
op.execute("DELETE FROM note_embeddings")
op.execute("ALTER TABLE note_embeddings DROP CONSTRAINT note_embeddings_pkey")
op.execute("ALTER TABLE note_embeddings DROP COLUMN chunk_index")
op.execute("ALTER TABLE note_embeddings DROP COLUMN chunk_text")
op.execute("ALTER TABLE note_embeddings DROP COLUMN chunker_version")
op.execute("ALTER TABLE note_embeddings ADD PRIMARY KEY (note_id)")
-130
View File
@@ -1,130 +0,0 @@
"""Forge connections move to the user level
Revision ID: 0078
Revises: 0077
Create Date: 2026-08-19
A forge token is a user's credential, not an instance's: the single
admin-settings config meant every user's snippet-freshness and coverage reads
ran under the operator's token. Each user now owns a keyring of connections —
one per forge host — and projects resolve forge reads on their OWNER's
keyring, with an optional per-project pin (projects.forge_connection_id).
The data move carries the existing admin config into a connection row for the
first admin user (host parsed from the base URL), then deletes the old
setting keys outright — no legacy dual-read. The env-var channel
(FORGE_KIND/FORGE_BASE_URL/FORGE_TOKEN) is untouched by this migration; it
survives as an implicit keyring entry for admin users only.
"""
from urllib.parse import urlsplit
import sqlalchemy as sa
from alembic import op
revision = "0078"
down_revision = "0077"
branch_labels = None
depends_on = None
_SETTING_KEYS = ("forge_kind", "forge_base_url", "forge_token")
def upgrade() -> None:
op.create_table(
"forge_connections",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"user_id",
sa.Integer(),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("base_url", sa.Text(), nullable=False),
sa.Column("host", sa.Text(), nullable=False),
sa.Column("token", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("user_id", "host", name="uq_forge_connections_user_host"),
)
op.add_column(
"projects",
sa.Column(
"forge_connection_id",
sa.BigInteger(),
sa.ForeignKey(
"forge_connections.id",
ondelete="SET NULL",
name="fk_projects_forge_connection_id",
),
nullable=True,
),
)
# Data move: the admin-settings config becomes the first admin's keyring
# row. All three values must be present — a partial config never produced
# an adapter, so carrying it over would invent a connection that never
# worked.
conn = op.get_bind()
row = conn.execute(
sa.text(
"SELECT s.key, s.value FROM settings s"
" JOIN users u ON u.id = s.user_id"
" WHERE u.role = 'admin' AND s.key IN :keys"
" AND s.user_id = ("
" SELECT MIN(id) FROM users WHERE role = 'admin'"
" )"
).bindparams(sa.bindparam("keys", expanding=True)),
{"keys": list(_SETTING_KEYS)},
).fetchall()
values = {key: (value or "").strip() for key, value in row}
kind = values.get("forge_kind", "").lower()
base_url = values.get("forge_base_url", "").rstrip("/")
token = values.get("forge_token", "")
host = (urlsplit(base_url).hostname or "").lower()
if kind and base_url and token and host:
conn.execute(
sa.text(
"INSERT INTO forge_connections"
" (user_id, kind, base_url, host, token, created_at, updated_at)"
" SELECT MIN(id), :kind, :base_url, :host, :token, NOW(), NOW()"
" FROM users WHERE role = 'admin'"
),
{"kind": kind, "base_url": base_url, "host": host, "token": token},
)
conn.execute(
sa.text(
"DELETE FROM settings WHERE key IN :keys"
).bindparams(sa.bindparam("keys", expanding=True)),
{"keys": list(_SETTING_KEYS)},
)
def downgrade() -> None:
# Reverse data move: the first admin's row (if any) becomes the admin
# settings again. Other users' rows have no pre-0078 representation and
# are dropped with the table.
conn = op.get_bind()
row = conn.execute(
sa.text(
"SELECT user_id, kind, base_url, token FROM forge_connections"
" WHERE user_id = (SELECT MIN(id) FROM users WHERE role = 'admin')"
" ORDER BY id LIMIT 1"
)
).fetchone()
if row is not None:
for key, value in (
("forge_kind", row.kind),
("forge_base_url", row.base_url),
("forge_token", row.token),
):
conn.execute(
sa.text(
"INSERT INTO settings (user_id, key, value)"
" VALUES (:uid, :key, :value)"
" ON CONFLICT (user_id, key) DO UPDATE SET value = :value"
),
{"uid": row.user_id, "key": key, "value": value},
)
op.drop_column("projects", "forge_connection_id")
op.drop_table("forge_connections")
@@ -1,66 +0,0 @@
"""The shape ledger: code_shapes
Revision ID: 0079
Revises: 0078
Create Date: 2026-08-19
The accounting half of the pattern system: the snippet
library records canon (small); this table accounts for EVERY shape the
coverage extractor finds in a bound repo (total). Rows arrive `unclassified`
from the coverage sync and gain judgments — canonical / instance /
variant / exempt — from audits, hooks, and the mechanical proposer.
Unclassified IS the todo list.
"""
import sqlalchemy as sa
from alembic import op
revision = "0079"
down_revision = "0078"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"code_shapes",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"project_id",
sa.Integer(),
sa.ForeignKey("projects.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("repo_key", sa.Text(), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("symbol", sa.Text(), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("status", sa.Text(), nullable=False, server_default="unclassified"),
sa.Column(
"snippet_id",
sa.BigInteger(),
sa.ForeignKey("notes.id", ondelete="SET NULL"),
nullable=True,
),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("classified_by", sa.Text(), nullable=True),
sa.Column("classified_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("first_seen_commit", sa.Text(), nullable=False, server_default=""),
sa.Column("last_seen_commit", sa.Text(), nullable=False, server_default=""),
sa.Column("vanished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"project_id", "repo_key", "path", "symbol", "kind",
name="uq_code_shapes_identity",
),
)
op.create_index(
"ix_code_shapes_project_status", "code_shapes", ["project_id", "status"]
)
op.create_index("ix_code_shapes_snippet", "code_shapes", ["snippet_id"])
def downgrade() -> None:
op.drop_index("ix_code_shapes_snippet", table_name="code_shapes")
op.drop_index("ix_code_shapes_project_status", table_name="code_shapes")
op.drop_table("code_shapes")
@@ -1,54 +0,0 @@
"""Shape fingerprints + the mechanical proposer's columns
Revision ID: 0080
Revises: 0079
Create Date: 2026-08-21
Two additions to the ledger. `signature` / `body_sha` fingerprint each shape
(definition line + a whitespace/comment-insensitive hash of its block) so the
proposer can match on content and a later drift recheck can notice change,
without the ledger ever storing code. The proposal columns carry the
proposer's standing suggestion for an unclassified row — instance-of-#N with
a basis and score, or a derive-first group key — and `proposed_sha`
remembers the content it was judged at so a refresh re-examines only what
changed. Mechanical and recomputable: a restore that lacks them loses
nothing the next refresh does not rebuild.
"""
import sqlalchemy as sa
from alembic import op
revision = "0080"
down_revision = "0079"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("code_shapes", sa.Column("signature", sa.Text(), nullable=False, server_default=""))
op.add_column("code_shapes", sa.Column("body_sha", sa.Text(), nullable=False, server_default=""))
op.add_column(
"code_shapes",
sa.Column(
"proposed_snippet_id",
sa.BigInteger(),
sa.ForeignKey("notes.id", ondelete="SET NULL"),
nullable=True,
),
)
op.add_column("code_shapes", sa.Column("proposal_basis", sa.Text(), nullable=True))
op.add_column("code_shapes", sa.Column("proposal_score", sa.Float(), nullable=True))
op.add_column("code_shapes", sa.Column("proposal_group", sa.Text(), nullable=True))
op.add_column("code_shapes", sa.Column("proposed_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("code_shapes", sa.Column("proposed_sha", sa.Text(), nullable=False, server_default=""))
op.create_index(
"ix_code_shapes_proposed", "code_shapes", ["project_id", "proposed_snippet_id"]
)
def downgrade() -> None:
op.drop_index("ix_code_shapes_proposed", table_name="code_shapes")
for col in (
"proposed_sha", "proposed_at", "proposal_group", "proposal_score",
"proposal_basis", "proposed_snippet_id", "body_sha", "signature",
):
op.drop_column("code_shapes", col)
@@ -1,70 +0,0 @@
"""Shape history, recheck, and the divergence flag
Revision ID: 0081
Revises: 0080
Create Date: 2026-08-21
The payoff surface of the ledger. `classified_sha` remembers the fingerprint
a judgment was made at so a later body change under an instance/variant can
flag `recheck_at`; `diverges_from` is the button-B flag (a shape new since
the previous refresh, where one canon dominates its directory+kind, and not
proposed as that canon). `code_shape_events` is the what-was-used-when
record: every classification, vanish, reappearance, and drift as it
happened — history the row alone cannot keep once it moves on.
"""
import sqlalchemy as sa
from alembic import op
revision = "0081"
down_revision = "0080"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("code_shapes", sa.Column("classified_sha", sa.Text(), nullable=False, server_default=""))
op.add_column("code_shapes", sa.Column("recheck_at", sa.DateTime(timezone=True), nullable=True))
op.add_column(
"code_shapes",
sa.Column(
"diverges_from",
sa.BigInteger(),
sa.ForeignKey("notes.id", ondelete="SET NULL"),
nullable=True,
),
)
op.create_index("ix_code_shapes_diverges", "code_shapes", ["project_id", "diverges_from"])
op.create_table(
"code_shape_events",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"shape_id",
sa.Integer(),
sa.ForeignKey("code_shapes.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("project_id", sa.Integer(), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("symbol", sa.Text(), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("event", sa.Text(), nullable=False),
sa.Column("status", sa.Text(), nullable=True),
sa.Column("snippet_id", sa.BigInteger(), nullable=True),
sa.Column("classified_by", sa.Text(), nullable=True),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("commit", sa.Text(), nullable=False, server_default=""),
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_code_shape_events_shape", "code_shape_events", ["shape_id", "at"])
op.create_index(
"ix_code_shape_events_project_path", "code_shape_events", ["project_id", "path"]
)
def downgrade() -> None:
op.drop_index("ix_code_shape_events_project_path", table_name="code_shape_events")
op.drop_index("ix_code_shape_events_shape", table_name="code_shape_events")
op.drop_table("code_shape_events")
op.drop_index("ix_code_shapes_diverges", table_name="code_shapes")
for col in ("diverges_from", "recheck_at", "classified_sha"):
op.drop_column("code_shapes", col)
-26
View File
@@ -1,26 +0,0 @@
"""Per-binding ref — the branch a project's ledger follows
Revision ID: 0082
Revises: 0081
Create Date: 2026-08-21
Without a ref, a repo binding implies the repo's default branch, so the shape
ledger only sees work after a merge — while a project may do its work on a
dev branch. `ref` names the branch the coverage refresh reads; NULL means the
forge's default branch.
"""
import sqlalchemy as sa
from alembic import op
revision = "0082"
down_revision = "0081"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("repo_bindings", sa.Column("ref", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("repo_bindings", "ref")
@@ -1,26 +0,0 @@
"""Exempt/variant reason codes — a small fixed catalogue beside the prose
Revision ID: 0083
Revises: 0082
Create Date: 2026-08-21
The 2026-08 audit wrote the same free-text reason thousands of times
("scoped rule — styles one element of this view"); a judgment's WHY stays
prose, but an optional code from a fixed catalogue makes the ledger
filterable and aggregable ("how many pure helpers, how many test helpers").
"""
import sqlalchemy as sa
from alembic import op
revision = "0083"
down_revision = "0082"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("code_shapes", sa.Column("reason_code", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("code_shapes", "reason_code")
-40
View File
@@ -1,40 +0,0 @@
"""code_shape_uses — consumption edges, separate from conformance
Revision ID: 0084
Revises: 0083
Create Date: 2026-08-21
A ledger row carries ONE snippet_id: what shape this is (instance/variant of
a canon). But a shape can also CALL several canonical helpers — e.g. a
service function both conforming to the service-function convention and
consuming hash_token. The 2026-08 audit had to pick one; hook evidence
("pulled #N then wrote code referencing it") was stamped as instance when it
is a uses fact. This table holds the many-valued relation: shape → snippet,
with the basis and the evidence. Cascades with the shape and the snippet.
"""
import sqlalchemy as sa
from alembic import op
revision = "0084"
down_revision = "0083"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"code_shape_uses",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("shape_id", sa.Integer(), sa.ForeignKey("code_shapes.id", ondelete="CASCADE"), nullable=False),
sa.Column("snippet_id", sa.Integer(), sa.ForeignKey("notes.id", ondelete="CASCADE"), nullable=False),
sa.Column("basis", sa.Text(), nullable=False),
sa.Column("evidence", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.UniqueConstraint("shape_id", "snippet_id", name="uq_code_shape_uses_shape_snippet"),
)
op.create_index("ix_code_shape_uses_snippet", "code_shape_uses", ["snippet_id"])
def downgrade() -> None:
op.drop_index("ix_code_shape_uses_snippet", table_name="code_shape_uses")
op.drop_table("code_shape_uses")
@@ -1,74 +0,0 @@
"""Project inception: the decision record + always-on rulebook exclusions
Revision ID: 0085
Revises: 0084
Create Date: 2026-08-22
`projects.inception` is the WHY a project inherits what it does — NULL until
someone decides, at which point enter_project stops asking. The new
association `project_rulebook_exclusions` is the opt-out of a whole always-on
rulebook for one project (the sibling of the rule/topic suppressions).
Backfill: every project that exists when this runs is stamped
via="legacy" with its CURRENT standing (no exclusions, its subscriptions,
its design_system_id, no seed) — so the ask fires only for projects created
after the step shipped, and nothing a running install relies on changes.
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "0085"
down_revision = "0084"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"projects",
sa.Column("inception", postgresql.JSONB(), nullable=True),
)
op.create_table(
"project_rulebook_exclusions",
sa.Column(
"project_id", sa.BigInteger(),
sa.ForeignKey("projects.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
"rulebook_id", sa.BigInteger(),
sa.ForeignKey("rulebooks.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("now()"), nullable=False,
),
)
# Legacy stamp: what each existing project inherits today, recorded as a
# decision so the inception ask does not fire on a project that has been
# running for months.
op.execute(sa.text("""
UPDATE projects p SET inception = jsonb_build_object(
'via', 'legacy',
'decided_at', to_jsonb(now()),
'decided_by', NULL,
'choices', jsonb_build_object(
'exclude_always_on_rulebooks', '[]'::jsonb,
'subscribe_rulebooks', COALESCE(
(SELECT jsonb_agg(s.rulebook_id ORDER BY s.rulebook_id)
FROM project_rulebook_subscriptions s
WHERE s.project_id = p.id),
'[]'::jsonb),
'design_system_id', to_jsonb(p.design_system_id),
'seed_systems', false
)
)
WHERE p.inception IS NULL
"""))
def downgrade() -> None:
op.drop_table("project_rulebook_exclusions")
op.drop_column("projects", "inception")
@@ -1,35 +0,0 @@
"""code_shape_consumers — the CSS consumer map
Revision ID: 0086
Revises: 0085
Create Date: 2026-08-23
CSS is watched by name, by recipe, by token and by WHAT USES IT. This table
holds the fourth: CSS shape → the file whose markup names its class, with how
many times. Mechanical and recomputed by every coverage sync from the repo
archive; the analogue of code_shape_uses for styling. Cascades with the shape.
"""
import sqlalchemy as sa
from alembic import op
revision = "0086"
down_revision = "0085"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"code_shape_consumers",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("shape_id", sa.Integer(), sa.ForeignKey("code_shapes.id", ondelete="CASCADE"), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("count", sa.Integer(), nullable=False, server_default="1"),
sa.Column("basis", sa.Text(), nullable=False, server_default="template"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.UniqueConstraint("shape_id", "path", name="uq_code_shape_consumers_shape_path"),
)
def downgrade() -> None:
op.drop_table("code_shape_consumers")
-109
View File
@@ -1,109 +0,0 @@
"""canonical_systems — the global area vocabulary, promoted from a constant
to a table
Revision ID: 0087
Revises: 0086
Create Date: 2026-08-26
The eight standard area names already existed as `STANDARD_SYSTEMS`, a tuple in
services/systems.py that is seeded into a project at inception. A
constant cannot be referenced: a rule that applies across projects has nothing
to point at, because `systems.project_id` is NOT NULL and a family rule cannot
be chained to one project's row. This makes the vocabulary a table so it can be
a foreign key, and adds the nullable `systems.canonical_id` that maps a
project's local System onto it.
Deliberately no `user_id`: the catalog is GLOBAL so a shared project inherits
the vocabulary rather than re-earning it. `record_systems` is untouched — it
joins note_id/system_id and never sees this table, so no association data
moves, and no System's own `name` is rewritten.
The seed rows are written here verbatim rather than imported from the service:
a migration is a historical record and must keep running unchanged after the
service's list moves on.
"""
import sqlalchemy as sa
from alembic import op
revision = "0087"
down_revision = "0086"
branch_labels = None
depends_on = None
# (name, slug, description) — the milestone-297 vocabulary, with the slug the
# service computes (canonical_slug: lowercase, "&" -> "and", non-alphanumerics
# collapsed to "-"). Charters stay generic on purpose: a project refines its
# own System's description, never this one. Nothing here names an app, a repo,
# a vendor or a house convention — the catalog ships to every install.
_SEED = (
("CI & Release", "ci-and-release",
"How the project is verified and shipped: pipelines, runners, image/artifact builds, release tagging and rollback."),
("Auth & Access", "auth-and-access",
"Who may do what: identity, sessions/tokens, permissions and the scoping of every read and write to the right users."),
("Data Model & Storage", "data-model-and-storage",
"What is stored and how it is shaped: the schema, migrations, serialisation and the services that own a table's lifecycle."),
("API Surface", "api-surface",
"The doors into the capability: HTTP routes, tool/RPC surfaces, request parsing, error envelopes and their contracts."),
("UI & Design", "ui-and-design",
"What people see and touch: views, components, client state, and the design tokens/recipes they are built from."),
("Import & Export", "import-and-export",
"Data crossing the boundary: backups, exports, imports, sync with other systems, file formats."),
("Background Jobs", "background-jobs",
"Work that runs without a request: schedulers, queues, periodic ticks, retention and maintenance."),
("Observability", "observability",
"How the system reports on itself: logging, metrics, audit trails, health and diagnostics."),
)
def upgrade() -> None:
canonical_systems = op.create_table(
"canonical_systems",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("name", sa.Text(), nullable=False),
sa.Column("slug", sa.Text(), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("order_index", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("deleted_batch_id", sa.Text(), nullable=True),
)
# Unique among LIVE rows only, so a soft-deleted entry doesn't block
# recreating or restoring the same area (the rules/topics convention).
op.create_index(
"uq_canonical_systems_slug", "canonical_systems", ["slug"],
unique=True, postgresql_where=sa.text("deleted_at IS NULL"),
)
op.bulk_insert(
canonical_systems,
[
{"name": name, "slug": slug, "description": description, "order_index": index}
for index, (name, slug, description) in enumerate(_SEED)
],
)
op.add_column(
"systems",
sa.Column("canonical_id", sa.Integer(), nullable=True),
)
# SET NULL, not CASCADE: retiring a catalog entry must never delete a
# project's System along with it.
op.create_foreign_key(
"fk_systems_canonical_id", "systems", "canonical_systems",
["canonical_id"], ["id"], ondelete="SET NULL",
)
op.create_index("ix_systems_canonical_id", "systems", ["canonical_id"])
# Existing Systems are left UNMAPPED on purpose. An exact-slug match would
# be safe, but a near miss ("CI & runners" vs "CI & Release") is a judgment
# call — those go through the propose/confirm path so a human approves each
# one, rather than being decided by a migration nobody reviews.
def downgrade() -> None:
op.drop_index("ix_systems_canonical_id", table_name="systems")
op.drop_constraint("fk_systems_canonical_id", "systems", type_="foreignkey")
op.drop_column("systems", "canonical_id")
op.drop_index("uq_canonical_systems_slug", table_name="canonical_systems")
op.drop_table("canonical_systems")
@@ -1,104 +0,0 @@
"""rules gain a trigger, a tier, canon tags and typed edges
Revision ID: 0088
Revises: 0087
Create Date: 2026-08-26
A rule could not say WHEN it applies, WHICH area it is about, or WHAT other
rule it belongs with. All three were being written as prose instead — a
project's System description restating rule text, a rule's `why` naming the
note that caused it, and two halves of one shape merged into a single row
because either could surface without the other.
Four additions, each replacing something that was already being said in words:
- `when_to_apply` — the trigger. Nullable HERE and required at the service
layer, because existing rules have none and a migration cannot invent one.
- `tier` — `always_on` (preloaded, as everything is today) or `conditional`
(reachable, surfaced when its trigger fires). Defaults to `always_on`, so
this migration changes NOTHING about which rules bind: an install upgrades
and every rule keeps arriving exactly as it did.
- `arose_from_id` — the record that caused the rule, the edge notes and tasks
already have.
- `rule_systems` / `rule_relations` — the canon tag and the typed edges.
"""
import sqlalchemy as sa
from alembic import op
revision = "0088"
down_revision = "0087"
branch_labels = None
depends_on = None
# Kept in one place so upgrade and the CHECK agree by construction (a
# whitelisted value means DROP + ADD CONSTRAINT in the same migration —
# there is no prior constraint here, so the pair is created together).
_TIERS = ("always_on", "conditional")
_RELATION_KINDS = ("co_surfaces", "overrides", "elaborates")
def _in_list(column: str, values: tuple[str, ...]) -> str:
return f"{column} IN (" + ", ".join(f"'{v}'" for v in values) + ")"
def upgrade() -> None:
op.add_column("rules", sa.Column("when_to_apply", sa.Text(), nullable=True))
op.add_column(
"rules",
sa.Column("tier", sa.Text(), nullable=False, server_default="always_on"),
)
op.create_check_constraint("ck_rules_tier", "rules", _in_list("tier", _TIERS))
# SET NULL, not CASCADE: the record that prompted a rule can be trashed
# without taking the rule with it — provenance is a claim about history,
# and losing the source does not repeal the rule.
op.add_column("rules", sa.Column("arose_from_id", sa.BigInteger(), nullable=True))
op.create_foreign_key(
"fk_rules_arose_from_id", "rules", "notes",
["arose_from_id"], ["id"], ondelete="SET NULL",
)
# Which global AREA a rule is about. Points at the canonical catalog, never
# at a project's `systems` row — a rule that spans projects cannot be
# chained to one project's vocabulary (0087).
op.create_table(
"rule_systems",
sa.Column("rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True),
sa.Column("canonical_id", sa.Integer(), sa.ForeignKey("canonical_systems.id", ondelete="CASCADE"), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_index("ix_rule_systems_canonical_id", "rule_systems", ["canonical_id"])
# Typed edges between rules. Each kind exists because its absence forced a
# workaround: co_surfaces (merging two rules into one row), overrides (a
# stricter project rule written as a duplicate), elaborates (a local
# addendum sitting beside its parent with nothing to say it is one).
op.create_table(
"rule_relations",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column("from_rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), nullable=False),
sa.Column("to_rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("note", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.CheckConstraint(_in_list("kind", _RELATION_KINDS), name="ck_rule_relations_kind"),
# A rule cannot relate to itself, and one pair carries a given kind
# once — a second row would surface the same rule twice.
sa.CheckConstraint("from_rule_id <> to_rule_id", name="ck_rule_relations_not_self"),
sa.UniqueConstraint("from_rule_id", "to_rule_id", "kind", name="uq_rule_relations_edge"),
)
op.create_index("ix_rule_relations_from", "rule_relations", ["from_rule_id"])
op.create_index("ix_rule_relations_to", "rule_relations", ["to_rule_id"])
def downgrade() -> None:
op.drop_index("ix_rule_relations_to", table_name="rule_relations")
op.drop_index("ix_rule_relations_from", table_name="rule_relations")
op.drop_table("rule_relations")
op.drop_index("ix_rule_systems_canonical_id", table_name="rule_systems")
op.drop_table("rule_systems")
op.drop_constraint("fk_rules_arose_from_id", "rules", type_="foreignkey")
op.drop_column("rules", "arose_from_id")
op.drop_constraint("ck_rules_tier", "rules", type_="check")
op.drop_column("rules", "tier")
op.drop_column("rules", "when_to_apply")
-57
View File
@@ -1,57 +0,0 @@
"""rule_embeddings — rules become findable by meaning
Revision ID: 0089
Revises: 0088
Create Date: 2026-08-26
Rules were the only major record type with no vector, so `search` could never
return one and a rule could only ever arrive by being preloaded. That single
fact is what made every rule compete for the same always-on budget.
A sibling table rather than a generalisation of note_embeddings: the row could
have been made polymorphic, but the SEARCH could not — semantic_search_notes is
Note-specific scoping end to end, and a rule shares none of it. See the model
docstring for the full reasoning.
The vectors are DERIVED data. Nothing is backfilled here: the startup backfill
regenerates them, which is also how a chunker-version bump is handled.
"""
import sqlalchemy as sa
from alembic import op
revision = "0089"
down_revision = "0088"
branch_labels = None
depends_on = None
# Matches note_embeddings — bge-small-en-v1.5, 384-dim unit-normalized.
_EMBEDDING_DIM = 384
def upgrade() -> None:
op.create_table(
"rule_embeddings",
sa.Column("rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True),
sa.Column("chunk_index", sa.Integer(), primary_key=True),
sa.Column("chunk_text", sa.Text(), nullable=False),
sa.Column("chunker_version", sa.Integer(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
# The vector column is added by raw DDL for the same reason 0067 did it:
# the type comes from the pgvector extension, not from SQLAlchemy's
# type system.
op.execute(f"ALTER TABLE rule_embeddings ADD COLUMN embedding vector({_EMBEDDING_DIM}) NOT NULL")
# HNSW for cosine distance — matches Vector.cosine_distance (`<=>`), so the
# search is an indexed ORDER BY ... LIMIT k rather than a full scan.
op.execute(
"""
CREATE INDEX ix_rule_embeddings_embedding_hnsw
ON rule_embeddings
USING hnsw (embedding vector_cosine_ops)
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_rule_embeddings_embedding_hnsw")
op.drop_table("rule_embeddings")
@@ -1,62 +0,0 @@
"""a rule can carry its own check — verify_with, expires_when, verified_at
Revision ID: 0090
Revises: 0089
Create Date: 2026-08-27
A rulebook holds two kinds of row in one table. A NORM is a decision: it has
no truth value, and it changes only when its author changes it — which they
know they did. A CONSTRAINT is a fact about someone else's software: a
runner's shell, a bot's config, a tool that exists. Nobody is present when
that goes false.
What goes stale in a rulebook is the constraints, not the norms — and a
stale constraint keeps instructing every session that loads it, long after
the world it described has moved.
Three nullable columns, so a rule can say how to check itself:
- `verify_with` — how to tell whether this is still true. A command, a path,
a URL, a query. Prose is allowed; something runnable is better.
- `expires_when` — the STATE under which it stops being true. Deliberately
not a date: constraints do not expire on a schedule, they expire when the
world underneath them moves.
- `verified_at` — when the check last passed. NULL means never checked, and
sorts FIRST in the sweep: unexamined outranks examined-long-ago.
All three nullable and all three optional, because most rules should set
none of them. A null `verify_with` is not an omission — it is the honest
marker of "this one is a decision, and there is nothing to go and check."
That signal only works if the field stays empty wherever it belongs empty.
No CHECK constraint is involved. Nothing is
backfilled: a migration cannot invent a check any more than 0088 could
invent a trigger.
"""
import sqlalchemy as sa
from alembic import op
revision = "0090"
down_revision = "0089"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("rules", sa.Column("verify_with", sa.Text(), nullable=True))
op.add_column("rules", sa.Column("expires_when", sa.Text(), nullable=True))
op.add_column(
"rules",
sa.Column("verified_at", sa.DateTime(timezone=True), nullable=True),
)
# No index on (verify_with, verified_at). The sweep this exists for reads
# an operator's whole rulebook — hundreds of rows, not millions — and runs
# when a human asks for it, never on a request path. An index here would
# be maintained on every rule write to serve a query that a sequential
# scan answers instantly.
def downgrade() -> None:
op.drop_column("rules", "verified_at")
op.drop_column("rules", "expires_when")
op.drop_column("rules", "verify_with")
-65
View File
@@ -1,65 +0,0 @@
"""task_kind gains 'spike' — the investigation, not the change
Revision ID: 0091
Revises: 0090
Create Date: 2026-08-27
A spike is a task shape the others cannot hold. `work` ships a change;
`issue` fixes something broken. A spike is time-boxed and its output is
KNOWLEDGE — it succeeds by producing an answer, and nothing ships at the
end of it. "Find out whether the runner can be given a bash shell" is not
work, and filing it as work makes a finished investigation look like an
abandoned change.
It is the record a failed check asks for. Rules carry a
`verify_with`; when one of those fails, the rule is wrong and the next move
is often to go and find out what replaced it. `notes.arose_from_id` already
exists (0065), so that constraint -> spike link needs no further schema.
`task_kind` is gated by a CHECK whitelist, so the value and the
widened constraint land in the SAME migration — DROP then ADD, exactly as
0065 did when it introduced 'issue'. Adding the value and constraining it
later leaves a window where the database accepts anything.
'plan' stays in the list though it is retired (plans are milestones since
0066): historical plan-tasks still carry it, and dropping it from the
whitelist would make old rows unwritable.
"""
from alembic import op
revision = "0091"
down_revision = "0090"
branch_labels = None
depends_on = None
# One tuple so the upgrade and the downgrade cannot disagree about what the
# list was on either side of this migration.
_KINDS_AFTER = ("work", "plan", "issue", "spike")
_KINDS_BEFORE = ("work", "plan", "issue")
# Restated rather than imported from 0088, which has the same helper. A
# migration is a snapshot: it must keep working when the code around it has
# moved on, so it never imports from live modules or from its siblings. Six
# duplicated lines are the price of that, and the cheap half of the bargain.
def _in_list(values: tuple[str, ...]) -> str:
return "task_kind IN (" + ", ".join(f"'{v}'" for v in values) + ")"
def upgrade() -> None:
op.drop_constraint("notes_task_kind_check", "notes", type_="check")
op.create_check_constraint(
"notes_task_kind_check", "notes", _in_list(_KINDS_AFTER),
)
def downgrade() -> None:
# Any row already filed as a spike would violate the narrowed constraint,
# so they are demoted to 'work' first. Lossy and deliberately so: the
# alternative is a downgrade that fails on real data, which is worse than
# a downgrade that says what it did.
op.execute("UPDATE notes SET task_kind = 'work' WHERE task_kind = 'spike'")
op.drop_constraint("notes_task_kind_check", "notes", type_="check")
op.create_check_constraint(
"notes_task_kind_check", "notes", _in_list(_KINDS_BEFORE),
)
@@ -1,77 +0,0 @@
"""a note can carry its own check — verify_with, expires_when, verified_at
Revision ID: 0092
Revises: 0091
Create Date: 2026-08-28
The sibling of 0090, which gave rules the same three columns. Same
distinction, one table over:
A NORM is a decision — no truth value, and it changes only when its author
changes it, which they know they did. A CONSTRAINT is a fact about someone
else's software, and nobody is present when it goes false.
Notes hold far more constraints than rules do, and hold them for longer. A
cross-project reference note asserting what a signing service does on a
duplicate upload, or how a forge numbers its CI runs, is believed by every
project that reads it, and there is nothing in the record that says when
anyone last looked. `note_supersessions` only fires once a human has read
the note, disagreed, and written the correction — which is the case where
the note was already believed.
Three nullable columns:
- `verify_with` — how to tell whether this is still true. A command, a path,
a URL, a query. Prose is allowed; something runnable is better.
- `expires_when` — the STATE under which it stops being true. Deliberately
not a date: constraints do not expire on a schedule, they expire when the
world underneath them moves.
- `verified_at` — when the check last passed. NULL means never checked, and
sorts FIRST in the sweep: unexamined outranks examined-long-ago.
WHICH ROWS THESE ARE FOR. `notes` is one table holding notes, tasks,
snippets and processes, so these columns land on all of them. Only non-task,
non-snippet records are OFFERED them (gated at the service): a task's decay is its status, and a snippet
already carries a richer, location-aware verdict in `data.verification`. The
columns exist on the other rows and stay null there; a gate that lives in
the schema would have meant a partial index or a CHECK across three columns
to express something the write path can say in two lines.
All three optional, because most notes should set none of them — the whole
value of the sweep is that its output is short. A null `verify_with` is not
an omission; it is the honest marker of "this one is a decision, and there
is nothing to go and check."
No CHECK constraint is involved. Nothing is
backfilled: a migration cannot invent a check.
"""
import sqlalchemy as sa
from alembic import op
revision = "0092"
down_revision = "0091"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("notes", sa.Column("verify_with", sa.Text(), nullable=True))
op.add_column("notes", sa.Column("expires_when", sa.Text(), nullable=True))
op.add_column(
"notes",
sa.Column("verified_at", sa.DateTime(timezone=True), nullable=True),
)
# No index, for 0090's reason — the sweep runs when a human asks, never on
# a request path — but the margin is thinner here and worth naming. `rules`
# is hundreds of rows; `notes` is thousands and grows with every session.
#
# Still a sequential scan's job at this size, and an index on
# (verified_at) filtered to `verify_with IS NOT NULL` would be maintained
# on every note write to serve one operator-initiated query. If a real
# plan says otherwise, add it then rather than guessing here.
def downgrade() -> None:
op.drop_column("notes", "verified_at")
op.drop_column("notes", "expires_when")
op.drop_column("notes", "verify_with")
-80
View File
@@ -1,80 +0,0 @@
"""rules gain an edit history — rule_versions
Revision ID: 0093
Revises: 0092
Create Date: 2026-08-29
The sibling `note_versions` has had for a long time. A note's every meaningful
edit is snapshotted, and the design-system note calls that history "the
changelog". A RULE — which binds behaviour on every session that loads it —
had nothing: an edit destroyed what it used to say, with no record anywhere,
and keeping a superseded statement meant a person remembering to copy it
somewhere. The more consequential record had the weaker protection.
Three things are deliberately NOT copied from note_versions, and each is a
guard that exists there for a reason that does not hold here:
- **No pruning, and no MAX_VERSIONS.** That cap defends against note autosave
filling every slot. Rules have no autosave; every edit is a deliberate
update_rule. A rule is edited a handful of times in its life, and capping
invites losing the one edit somebody needed.
- **No pin columns.** `pin_kind`/`pin_label` exist so a note's version can
survive that pruning. With nothing pruning, a pin protects a row that was
never at risk.
- **No minimum interval.** 300 seconds between snapshots is also an autosave
defence; here it would only ever discard a second deliberate edit.
`user_id` is the ACTOR rather than the owner, and is SET NULL rather than
CASCADE: deleting a user must not erase the history of the rules they edited.
The edit still happened and the rule still binds because of it.
No CHECK constraint is involved. Nothing is backfilled — a
migration cannot invent the text a rule used to have, and inventing "the
current text, as of now" would be worse than an empty history, because it
would look like a record of an edit that never occurred.
"""
import sqlalchemy as sa
from alembic import op
revision = "0093"
down_revision = "0092"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"rule_versions",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column(
"rule_id",
sa.BigInteger(),
sa.ForeignKey("rules.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"user_id",
sa.BigInteger(),
sa.ForeignKey("users.id", ondelete="SET NULL"),
nullable=True,
),
sa.Column("title", sa.Text(), nullable=False, server_default=""),
sa.Column("statement", sa.Text(), nullable=False, server_default=""),
sa.Column("why", sa.Text(), nullable=True),
sa.Column("how_to_apply", sa.Text(), nullable=True),
sa.Column("when_to_apply", sa.Text(), nullable=True),
sa.Column("tier", sa.Text(), nullable=True),
sa.Column("verify_with", sa.Text(), nullable=True),
sa.Column("expires_when", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
# The only query this table serves is "the history of THIS rule, newest
# first" — unlike 0092's columns, which are read by an operator-initiated
# sweep over the whole set. Every read here is keyed on rule_id, so the
# index earns its write cost immediately rather than on a hunch.
op.create_index("ix_rule_versions_rule_id", "rule_versions", ["rule_id"])
def downgrade() -> None:
op.drop_index("ix_rule_versions_rule_id", table_name="rule_versions")
op.drop_table("rule_versions")
@@ -1,83 +0,0 @@
"""add rule_usage_events — was a surfaced rule ever read?
Revision ID: 0094
Revises: 0093
Create Date: 2026-09-02
The sibling `note_usage_events` has had since 0071, and the third rule-side
table to arrive after `rule_embeddings` and `rule_versions` — each one added
because the rule side kept inheriting machinery built for notes and getting
the weaker version of it.
WHAT IT MEASURES. A standing-rule arm that returns something on every call
and clears its threshold every time is either a perfectly tuned surface or a
bar it cannot fail to clear, and `retrieval_logs`
cannot tell them apart: it records what the ranker scored, never whether the
hint was any use.
WHY NOT A rule_id COLUMN ON note_usage_events. The row shares no note-specific
fields and the aggregate readout is the same shape, which is the strongest case
for sharing there is. What decides against it is identity at
RESTORE: `note_usage_events`'s importer maps `note_id` through `note_id_map`
and drops what does not resolve. A rule id parked in that column would come
back from a backup silently reattached to whatever note took that number —
telemetry not merely lost but wrong, and wrong in a way nothing downstream
could detect. `rule_versions` made the same call for the same reason.
FK-free on `rule_id` and `user_id`, matching note_usage_events, retrieval_logs
and app_logs — and deliberately unlike `rule_versions`, which does carry FKs.
The difference is what the row is for: a version belongs to a rule's history
and dies with it; telemetry outlives the row it describes. Deleting a rule must
not erase the evidence that it was surfaced forty times and opened never, since
that evidence is exactly the case for having deleted it.
No CHECK on `event`, matching the note twin. DROP + ADD governs adding a value
to a column that is already gated; it does not require gating one that never was,
and a two-member enum whose members are written by two functions in one module
is not where that discipline earns its cost.
Downgrade drops the table outright. The data is purely observational — nothing
reads it for correctness, so losing it costs history and no behaviour.
"""
from alembic import op
import sqlalchemy as sa
revision = "0094"
down_revision = "0093"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"rule_usage_events",
# BigInteger throughout where the note twin uses Integer: rules.id is
# BigInteger, so rule_id must be, and a high-churn append-only table is
# a poor place to discover an id ceiling.
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.text("now()"),
),
sa.Column("user_id", sa.BigInteger(), nullable=True),
sa.Column("rule_id", sa.BigInteger(), nullable=False),
sa.Column("event", sa.Text(), nullable=False),
sa.Column("source", sa.Text(), nullable=False),
)
# Every readout is "these rule ids, split by event", so the composite is the
# one that actually gets used; the others serve pruning and per-user views.
op.create_index(
"ix_rule_usage_rule_event", "rule_usage_events", ["rule_id", "event"]
)
op.create_index("ix_rule_usage_created_at", "rule_usage_events", ["created_at"])
op.create_index("ix_rule_usage_user_id", "rule_usage_events", ["user_id"])
def downgrade() -> None:
op.drop_index("ix_rule_usage_user_id", table_name="rule_usage_events")
op.drop_index("ix_rule_usage_created_at", table_name="rule_usage_events")
op.drop_index("ix_rule_usage_rule_event", table_name="rule_usage_events")
op.drop_table("rule_usage_events")
@@ -1,51 +0,0 @@
"""add retrieval_logs.suppressed_count — tell a ranker decline from a repeat
Revision ID: 0095
Revises: 0094
Create Date: 2026-09-03
`result_count == 0` has always meant "this surface said nothing", which is the
right number for "was the hint any use" and the wrong one for tuning a
threshold. It folds together two unrelated events:
- the ranker found nothing above the bar — the ONLY evidence a threshold is
set too high; and
- the ranker found something the session had already been shown — a decline
that says nothing whatever about the bar.
The rule arms filter in Python after the search, so they can count the second
kind exactly. The note arms pass `exclude_ids` INTO semantic_search_notes, so
the dropped rows never come back and there is nothing to count.
NULLABLE, AND THE NULL IS THE POINT. A surface that does not measure
suppression stores NULL, not 0, and the readout renders it as "not measured"
rather than "none". Defaulting to 0 would make an unmeasured surface look like
a perfectly clean one — the substitution of an artifact for a measurement
that this column exists to prevent.
No backfill for the same reason: existing rows genuinely do not know, and
saying so is the honest state. `retrieval_logs` is not restored from backup,
so no importer changes.
Downgrade drops the column. Purely observational — nothing reads it for
correctness.
"""
from alembic import op
import sqlalchemy as sa
revision = "0095"
down_revision = "0094"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"retrieval_logs",
sa.Column("suppressed_count", sa.Integer(), nullable=True),
)
def downgrade() -> None:
op.drop_column("retrieval_logs", "suppressed_count")
@@ -1,62 +0,0 @@
"""add retrieval_logs.best_available_score — the score the bar rejected
Revision ID: 0096
Revises: 0095
Create Date: 2026-09-08
`cleared_threshold` was documented as the number to read FIRST — "a surface
that clears its bar on nearly every call is either well-tuned or too loose,
and p10 says which". It was never a measurement. The search applies the
threshold before returning, so every returned result cleared the bar by
construction and a call with no results has no `top_score` to compare:
the condition is true exactly when `result_count > 0`.
`zero_result_calls + cleared_threshold == calls` held on all nineteen
source/window readings ever taken. It was `calls - zero_result_calls`
wearing a name that promised a second opinion, and a reading procedure was
built on top of it that asked the reader to compare a number against itself.
THE MISSING NUMBER, and the reason this is a column rather than a deletion.
The question the table exists to answer is "is the bar in the right place",
and that question is only answerable from the calls that returned NOTHING:
how close did the best rejected candidate come? A bar at 0.72 turning away
a stream of 0.71s is set too high by a hair. A bar turning away 0.30s is
doing its job. Those two are indistinguishable today — both render as a
zero-result call — and no arrangement of the existing columns separates
them, because the losing score is discarded inside the search.
So the searches now rank without the bar and apply it in Python, which
costs nothing (the rows were already ordered by distance, and the qualifying
set is provably identical — above-threshold rows sort first), and the best
score seen becomes observable.
NULLABLE, AND UNBACKFILLED, for the reason 0095 spells out: a row written
before this shipped genuinely does not know what its best rejected candidate
scored, and saying so is the honest state. A 0.0 default would read as "the
corpus had nothing remotely relevant" — an artifact standing in for a
measurement, which is the defect this column corrects.
`retrieval_logs` is not restored from backup, so no importer changes.
Downgrade drops the column. Purely observational — nothing reads it for
correctness.
"""
from alembic import op
import sqlalchemy as sa
revision = "0096"
down_revision = "0095"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"retrieval_logs",
sa.Column("best_available_score", sa.Float(), nullable=True),
)
def downgrade() -> None:
op.drop_column("retrieval_logs", "best_available_score")
@@ -1,60 +0,0 @@
"""add retrieval_logs.best_available_id — WHICH record the bar refused
Revision ID: 0097
Revises: 0096
Create Date: 2026-09-09
0096 added `best_available_score` so a threshold could be judged from what it
rejected. It records how CLOSE the bar came to firing and not WHAT it turned
away, and that turns out to be the half a decision actually needs.
Live, `pre_tool_rule` shows a bar of ~0.72 with a near-miss p90 of 0.7071 —
about 117 declines a day sitting within 0.013 of firing. Dropping the bar to
0.707 would take that arm from 22 hits a day to roughly 139, a six-fold change
on a surface that runs before every Bash call. The percentile says the mass is
there. Nothing says whether it is worth showing.
AND THE TWO OBVIOUS INSTRUMENTS DO NOT ANSWER IT. Pull-through cannot: the
injected rule line already carries title and trigger, so a session can comply
without ever calling `get_rule`, and rule pull-through therefore understates
usefulness by construction. Reading the rejected records can — and `result_ids`
holds only what was RETURNED, so on a zero-result call it is empty and the
near-missed record has no name.
So: the id, beside the score, from the SAME ranked candidate. The two must
never be able to describe different records — a score paired with its
neighbour's id would be worse than no id at all, because it invites a reader to
judge the wrong record and conclude the bar is fine.
NULLABLE AND UNBACKFILLED, for the reason 0095 and 0096 both spell out: a row
written before this genuinely does not know, and inventing a value would put an
artifact where a measurement belongs. Null here means "not measured", never
"nothing was close".
NOT A FOREIGN KEY, deliberately. `retrieval_logs` spans record types — the
rules arms store rule ids, the note arms store note ids — and `source` is what
says which table an id belongs to, exactly as `result_ids` has always worked.
A constraint would have to point at one table and would be wrong for the other.
Downgrade drops the column. Purely observational — nothing reads it for
correctness.
"""
from alembic import op
import sqlalchemy as sa
revision = "0097"
down_revision = "0096"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"retrieval_logs",
sa.Column("best_available_id", sa.Integer(), nullable=True),
)
def downgrade() -> None:
op.drop_column("retrieval_logs", "best_available_id")
-90
View File
@@ -1,90 +0,0 @@
"""rules gain a `kind` — a preference is a rule that does not bind
Revision ID: 0098
Revises: 0097
Create Date: 2026-09-10
A rule says what must be followed. There was no way to record the other
thing the operator kept writing rules for: **how they want work done**.
Several rules in a mature rulebook are not really rules — pace this kind of
debugging, hand off an action with its reason, end a finding with an offer.
Ignoring one of those does not break anything or cross a boundary; it costs
consistency. They were written as rules because a rule was the only record
that is global, keyed to a situation, and delivered when that situation
arrives.
So: `kind`. `rule` binds. `preference` describes how this person wants it
done, and — the part that makes it its own kind rather than a softer label —
**it is expected to change as the work teaches it.** The agent updates a
preference in the ordinary course of working, where a rule waits for its
author.
WHY A COLUMN AND NOT A TABLE. Everything a preference needs already exists on
`rules` and nowhere else: `when_to_apply` as a real column, a
trigger-dominated embedding document, ownership-scoped search that is
deliberately not filtered to one project, three retrieval arms with per-arm
telemetry, typed relations, and versioning. The two differ in exactly one
dimension — force — and one dimension is a field.
The drift machinery is the decisive part. `rule_versions` already snapshots
every write, and `rule_relations.overrides` already models "this supersedes
that for its scope". A separate table would have rebuilt both, and moving
existing rows into it would have changed their ids — silently invalidating
every record in the corpus that cites a rule by number.
DEFAULTS TO `rule`, so this migration changes NOTHING about what binds. An
install upgrades and every existing row keeps the force it had. That is the
same reasoning 0088 used for `tier`, and it is the reason both are safe to
apply without reading the data first.
The CHECK is created with the column (there is no prior constraint,
so the pair is created together — a value added to it LATER does DROP + ADD
in one migration).
`rule_versions` GETS THE COLUMN TOO, and that half is not bookkeeping.
`record_if_changed` decides whether an edit is worth a snapshot by comparing
the fields a version carries; a field a version does not carry is a field
whose change records NO HISTORY AT ALL. Without this, turning a rule into a
preference — the single most consequential edit either kind can undergo,
because it is the moment something stops binding — would leave the history
silent. Nullable and no CHECK there, matching `tier`: a version is a record
of what was, and a constraint on it would refuse to store a kind later
dropped from the live whitelist.
Downgrade drops all three. Nothing reads `kind` for correctness; a rule that
was a preference simply becomes a rule again, which is the safe direction.
"""
import sqlalchemy as sa
from alembic import op
revision = "0098"
down_revision = "0097"
branch_labels = None
depends_on = None
# Kept in one place so upgrade and the CHECK agree by construction — 0088's
# idiom, for the same reason.
_KINDS = ("rule", "preference")
def _in_list(column: str, values: tuple[str, ...]) -> str:
return f"{column} IN (" + ", ".join(f"'{v}'" for v in values) + ")"
def upgrade() -> None:
op.add_column(
"rules",
sa.Column("kind", sa.Text(), nullable=False, server_default="rule"),
)
op.create_check_constraint("ck_rules_kind", "rules", _in_list("kind", _KINDS))
# Nullable, no CHECK — see the module docstring. A version written before
# this migration genuinely does not know, and NULL there means "not
# recorded", never "was a rule".
op.add_column("rule_versions", sa.Column("kind", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("rule_versions", "kind")
op.drop_constraint("ck_rules_kind", "rules", type_="check")
op.drop_column("rules", "kind")
@@ -1,124 +0,0 @@
"""scrub credential-shaped spans out of retrieval_logs.query
Revision ID: 0099
Revises: 0098
Create Date: 2026-09-11
`pre_tool_rule` retrieves against the RAW COMMAND TEXT and `write_path_rule`
against the code being written, so whatever was on the command line or in the
buffer is what `record_retrieval` wrote into `retrieval_logs.query`. A command
that exported a token stored the token.
`services/retrieval_telemetry.scrub_secrets` closes that going forward — the
value never reaches the column. It cannot reach BACKWARDS, and this does: it
rewrites the rows already written.
REDACTED IN PLACE, NOT DELETED. The rest of the row — score, threshold,
result count, duration, the near-miss record id — is legitimate evidence, and
it is what a threshold is tuned from. Deleting the row would throw that away
to remove a secret that lives in one column, so the column is what gets
rewritten. Rows with no credential in them are not touched at all.
THE PATTERNS ARE INLINED RATHER THAN IMPORTED, deliberately, against the DRY
instinct. A migration is a frozen record of a change that already happened on
every install that ran it; importing the live patterns would mean this
migration quietly does something different next year than it did when it ran,
and two installs at the same revision would no longer be in the same state.
The Python twin in `services/retrieval_telemetry.py` is free to grow — this is
what ran here, once. The one thing that must not drift is coverage, and the
guard for that is `test_retrieval_query_scrubbing.py`, which tests the live
function rather than this copy.
POSIX regex, not Python's. Postgres ARE supports the non-greedy `*?` the PEM
pattern needs, and `\\s`/`\\S`, so the shapes port directly. The `'gi'` flags
are global + case-insensitive, matching `re.sub` with `(?i)`.
NO BARE `auth` IN THE ASSIGNED PATTERN. It matches `--author=`, so a commit
naming an address would have had the address redacted — evidence eaten for a
word that only looks credential-shaped. `AUTH_TOKEN` is still caught, by
`token`.
Downgrade is a no-op, and honestly so: the original text is gone and a
migration cannot invent it back. Saying that plainly is better than a
downgrade that appears to restore something and does not.
"""
from alembic import op
revision = "0099"
down_revision = "0098"
branch_labels = None
depends_on = None
# Vendor-prefixed credentials — the prefix IS the tell, so no entropy guessing.
#
# `\m` IS LOAD-BEARING AND IS NOT `\b`. It anchors the prefix to the START OF
# A WORD, which the Python twin spells `\b`. Two separate mistakes were made
# porting this and they compounded:
#
# 1. The boundary was dropped entirely, so `sk-` matched inside any word
# containing it. `<task-notification>` — a string that appears in
# thousands of these rows — became `<ta[redacted:token]>`, because
# `sk-` + `notification` is a prefix followed by twelve word characters.
# 2. Writing `\b` would not have fixed it. In Postgres ARE `\b` is a
# BACKSPACE character, not a word boundary; `\m` (start of word) and
# `\y` (either edge) are the spellings that mean what Python's `\b`
# means.
#
# Both were live for one run of this migration, on one install, and the cost
# is recorded rather than papered over: the mangled rows cannot be restored,
# because the original text is what the UPDATE overwrote. The live scrubber in
# services/retrieval_telemetry.py was never affected — its `\b` is Python's
# and behaves correctly, which is why rows written after the deploy are intact
# and only migration-rewritten ones were damaged.
_TOKEN = (
r"\m(fmcp_|flt_|ghp_|gho_|ghs_|ghu_|github_pat_|glpat-|xox[abprs]-"
r"|sk-[A-Za-z0-9]*-?|AKIA|ASIA)[A-Za-z0-9_\-]{12,}"
)
# A value handed to a secret-NAMED variable, in shell, env, YAML, JSON or a
# query string. The NAME identifies it, so the value can be anything.
_ASSIGNED = (
r"\m([A-Za-z0-9_]*(token|secret|password|passwd|api[_-]?key|access[_-]?key)"
r"[A-Za-z0-9_]*)(\s*[:=]\s*[\"']?)([^\s\"'&]{8,})"
)
_AUTH_HEADER = r"(authorization\s*:\s*(bearer|basic|token)\s+)(\S+)"
_PEM = (
r"-----BEGIN [A-Z ]*PRIVATE KEY-----(.|\n)*?-----END [A-Z ]*PRIVATE KEY-----"
)
def _lit(pattern: str) -> str:
"""A regex as a SQL string literal.
A single quote inside a single-quoted SQL literal has to be DOUBLED, and
the assigned-value pattern contains two of them (it allows an optional
quote around the value). Left unescaped they close the literal early and
the migration dies on a syntax error — which is the whole reason this
helper exists rather than the patterns being pasted in inline.
"""
return pattern.replace("'", "''")
_SCRUB_SQL = f"""
UPDATE retrieval_logs
SET query = regexp_replace(
regexp_replace(
regexp_replace(
regexp_replace(query, '{_lit(_TOKEN)}', '[redacted:token]', 'gi'),
'{_lit(_ASSIGNED)}', '\\1\\3[redacted:assigned]', 'gi'),
'{_lit(_AUTH_HEADER)}', '\\1[redacted:auth-header]', 'gi'),
'{_lit(_PEM)}', '[redacted:private-key]', 'gi')
WHERE query IS NOT NULL
AND (query ~* '{_lit(_TOKEN)}'
OR query ~* '{_lit(_ASSIGNED)}'
OR query ~* '{_lit(_AUTH_HEADER)}'
OR query ~* '{_lit(_PEM)}')
"""
def upgrade() -> None:
op.execute(_SCRUB_SQL)
def downgrade() -> None:
"""Deliberately empty — the original text no longer exists to restore."""
@@ -1,96 +0,0 @@
"""drop the always-on tier: rules.tier, rulebooks.always_on, the exclusions table
Revision ID: 0100
Revises: 0099
Create Date: 2026-09-11
Every rule reaches a session by retrieval — because
something it is about to do made the rule relevant — and the machinery that
delivered rules unconditionally goes with it.
WHAT GOES, AND WHERE IT CAME FROM
- ``rules.tier`` and its ``ck_rules_tier`` CHECK (migration 0088). Dropping
the column takes the constraint with it. DROP + ADD is about ADDING a value to
a live whitelist, which needs DROP + ADD in the same migration; it does not
speak to removing the column outright, and saying so here is cheaper than
the next reader wondering whether it was forgotten.
- ``rule_versions.tier`` (migration 0098). A version records what a rule
SAID; with no tier on a rule there is nothing for a snapshot to carry.
- ``rulebooks.always_on`` (migration 0058). A rulebook reaches a project by
subscription now, and by nothing else.
- ``project_rulebook_exclusions`` (migration 0085). It recorded a project's
opt-out of an always-on rulebook. Opting out of something that no longer
binds you is not a state that can exist — declining a rulebook is
expressed by not subscribing to it.
IRREVERSIBLE, AND THE DOWNGRADE SAYS SO RATHER THAN PRETENDING
The downgrade recreates the columns and the table with their DEFAULTS. It
cannot restore WHICH rules were always-on, which rulebooks bound every project,
or which projects had opted out — that information is in what this drops.
That distinction is the one this repo keeps insisting on: a value invented to
fill a hole is not a measurement. So a downgraded database is structurally able
to run the old code and is NOT the database the old code was running against —
every rule comes back at the ``always_on`` default, which for the tier column
happens to mean "binding", the safe direction to be wrong in.
Anyone who needs the real prior state restores a backup taken before this ran.
"""
import sqlalchemy as sa
from alembic import op
revision = "0100"
down_revision = "0099"
branch_labels = None
depends_on = None
_TIERS = ("always_on", "conditional")
def _in_list(column: str, values: tuple[str, ...]) -> str:
return f"{column} IN (" + ", ".join(f"'{v}'" for v in values) + ")"
def upgrade() -> None:
op.drop_table("project_rulebook_exclusions")
op.drop_column("rulebooks", "always_on")
op.drop_column("rule_versions", "tier")
# The CHECK goes with the column it constrains; naming it here would be a
# second drop of the same object.
op.drop_column("rules", "tier")
def downgrade() -> None:
"""Structure only. See the module docstring — the values are gone."""
op.add_column(
"rules",
sa.Column("tier", sa.Text(), nullable=False, server_default="always_on"),
)
op.create_check_constraint("ck_rules_tier", "rules", _in_list("tier", _TIERS))
op.add_column("rule_versions", sa.Column("tier", sa.Text(), nullable=True))
op.add_column(
"rulebooks",
sa.Column(
"always_on", sa.Boolean(), nullable=False,
server_default=sa.text("false"),
),
)
op.create_table(
"project_rulebook_exclusions",
sa.Column(
"project_id", sa.BigInteger(),
sa.ForeignKey("projects.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
"rulebook_id", sa.BigInteger(),
sa.ForeignKey("rulebooks.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("now()"), nullable=True,
),
)
@@ -1,82 +0,0 @@
"""drop rulebook subscriptions and per-project suppressions
Revision ID: 0101
Revises: 0100
Create Date: 2026-09-15
A rule lives in a rulebook topic, where it is GLOBAL, or on one project, and
retrieval reads that home directly. Subscriptions were the last thing that
pretended a rulebook reached some projects and not others, and with no
always-on tier they changed nothing a session received — only what a
project's rule LISTING showed. Global and project-scoped rules cover what
subscriptions were for.
WHAT GOES
- ``project_rulebook_subscriptions`` (migration 0058).
- ``project_rule_suppressions`` and ``project_topic_suppressions``. They let a
project mute rules from a rulebook it subscribed to. With no subscription
there is nothing to mute; a project that departs from a global rule writes
a project rule with an ``overrides`` relation, which says why.
- The ``subscribe_rulebooks`` key inside ``projects.inception.choices``, and
the ``exclude_always_on_rulebooks`` key the retired always-on tier left
in the same place. Both describe decisions that can no longer be made; a stored
record carrying them would be read back as a choice the product offers.
IRREVERSIBLE, AND THE DOWNGRADE SAYS SO
The downgrade recreates the three tables empty. Which projects subscribed to
which rulebooks, and what they muted, is in what this drops. Restore a backup
taken before this ran if the prior state matters.
"""
import sqlalchemy as sa
from alembic import op
revision = "0101"
down_revision = "0100"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_table("project_topic_suppressions")
op.drop_table("project_rule_suppressions")
op.drop_table("project_rulebook_subscriptions")
op.execute(
"""
UPDATE projects
SET inception = jsonb_set(
inception, '{choices}',
(inception->'choices') - 'subscribe_rulebooks' - 'exclude_always_on_rulebooks'
)
WHERE inception IS NOT NULL
AND jsonb_typeof(inception->'choices') = 'object'
"""
)
def _join_table(name: str, other: str, other_table: str) -> None:
op.create_table(
name,
sa.Column(
"project_id", sa.BigInteger(),
sa.ForeignKey("projects.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
other, sa.BigInteger(),
sa.ForeignKey(f"{other_table}.id", ondelete="CASCADE"),
primary_key=True, nullable=False,
),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("now()"), nullable=True,
),
)
def downgrade() -> None:
"""Structure only. See the module docstring — the rows are gone."""
_join_table("project_rulebook_subscriptions", "rulebook_id", "rulebooks")
_join_table("project_rule_suppressions", "rule_id", "rules")
_join_table("project_topic_suppressions", "topic_id", "rulebook_topics")
@@ -1,59 +0,0 @@
"""milestone_embeddings — a plan becomes findable by meaning
Revision ID: 0102
Revises: 0101
Create Date: 2026-09-15
`search` covered notes, tasks and rules, and a milestone — the record a plan
lives in — could not be found at all. So "is there already a plan for this?"
had no tool, and a project whose roadmap was written as milestones had every
later plan opened as a new milestone beside the one that already described it.
The sibling of rule_embeddings (0089), for the reasons its model docstring
gives. The vectors are DERIVED: nothing is backfilled here, the startup
backfill writes them.
"""
import sqlalchemy as sa
from alembic import op
revision = "0102"
down_revision = "0101"
branch_labels = None
depends_on = None
# Matches note_embeddings and rule_embeddings — bge-small-en-v1.5, 384-dim.
_EMBEDDING_DIM = 384
def upgrade() -> None:
op.create_table(
"milestone_embeddings",
sa.Column(
"milestone_id", sa.Integer(),
sa.ForeignKey("milestones.id", ondelete="CASCADE"), primary_key=True,
),
sa.Column("chunk_index", sa.Integer(), primary_key=True),
sa.Column("chunk_text", sa.Text(), nullable=False),
sa.Column("chunker_version", sa.Integer(), nullable=False),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False,
server_default=sa.text("now()"),
),
)
# Raw DDL for the vector column, as 0067 and 0089 do: the type comes from
# the pgvector extension, not SQLAlchemy's type system.
op.execute(
f"ALTER TABLE milestone_embeddings ADD COLUMN embedding vector({_EMBEDDING_DIM}) NOT NULL"
)
op.execute(
"""
CREATE INDEX ix_milestone_embeddings_embedding_hnsw
ON milestone_embeddings
USING hnsw (embedding vector_cosine_ops)
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_milestone_embeddings_embedding_hnsw")
op.drop_table("milestone_embeddings")
@@ -1,71 +0,0 @@
"""retrieval_tuning_events — why a floor is where it is
Revision ID: 0103
Revises: 0102
Create Date: 2026-09-17
Similarity thresholds are not values somebody has to defend: the model that
reads the surface's own telemetry adjusts them, and the operator can still
touch them.
The number itself already has a home — the generic settings table. What has no
home is the ARGUMENT, and once the values move on their own the argument is the
part an operator needs: what changed, from what to what, who moved it, and on
what evidence. This table is that trail, and it is what makes the delegation
reviewable rather than merely automatic.
Nothing is backfilled. A surface with no rows here is sitting on its shipped
starting point, which is a true and useful thing for the history to say.
"""
import sqlalchemy as sa
from alembic import op
revision = "0103"
down_revision = "0102"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"retrieval_tuning_events",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
# FK-free, like retrieval_logs and app_logs: the record of why a number
# is where it is must outlive the account that moved it.
sa.Column("user_id", sa.Integer(), nullable=True),
# Also the surface's `retrieval_logs.source`, so a change can be read
# next to what the change did.
sa.Column("surface", sa.Text(), nullable=False),
sa.Column("dial", sa.Text(), nullable=False),
# Nullable: the first change to a surface has no stored predecessor. It
# moved off the shipped starting point, which is a different event from
# moving off a value somebody chose.
sa.Column("old_value", sa.Float(), nullable=True),
sa.Column("new_value", sa.Float(), nullable=False),
sa.Column(
"actor", sa.Text(), nullable=False, server_default=sa.text("'model'")
),
# Non-null here; non-BLANK is enforced at the service boundary, because
# a column that merely forbids NULL is satisfied by "" and a required
# field that accepts "" is a formality.
sa.Column("reason", sa.Text(), nullable=False),
)
# The only read this table has: one surface's history, newest first.
op.create_index(
"ix_retrieval_tuning_surface_created",
"retrieval_tuning_events",
["surface", sa.text("created_at DESC")],
)
def downgrade() -> None:
op.drop_index(
"ix_retrieval_tuning_surface_created", table_name="retrieval_tuning_events"
)
op.drop_table("retrieval_tuning_events")
@@ -1,51 +0,0 @@
"""retrieval_tuning_events carries the space each number was measured in
Revision ID: 0104
Revises: 0103
Create Date: 2026-09-17
A retrieval floor is a cosine distance in ONE embedding
model's geometry, computed over documents cut one particular way. Change the
model and every score moves at once; change the chunker and the same record
embeds different text. Either way a number chosen before the change is a
measurement of something that no longer exists — and today nothing records
which world it was chosen in, so the staleness is unknowable rather than
merely unknown.
`CHUNKER_VERSION` already solved exactly this for documents: stored per row, so
the startup backfill re-embeds precisely what is stale instead of wiping the
table. These two columns are that idea applied to the tuned numbers.
TWO COLUMNS, NOT ONE. A reader has to be able to say WHICH half
moved: a new embedding model and a re-cut document shape invalidate the same
numbers for different reasons, and a fused `"<model>@<n>"` could only report
that something changed.
NULLABLE, and not backfilled. The rows already in this table were written
under something, but naming it would be inventing a fact — the honest value is
"unstamped", which is a different answer from a model name that might be wrong.
`current_settings` reports an unstamped dial as exactly that.
"""
import sqlalchemy as sa
from alembic import op
revision = "0104"
down_revision = "0103"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"retrieval_tuning_events",
sa.Column("embedding_model", sa.Text(), nullable=True),
)
op.add_column(
"retrieval_tuning_events",
sa.Column("shape_version", sa.Integer(), nullable=True),
)
def downgrade() -> None:
op.drop_column("retrieval_tuning_events", "shape_version")
op.drop_column("retrieval_tuning_events", "embedding_model")
@@ -1,61 +0,0 @@
"""note_usage_events records WHICH project the reader was in
Revision ID: 0105
Revises: 0104
Create Date: 2026-09-20
`RetrievalLog` has carried `project_id` from the start, so
"this record was SURFACED on project B" has always been answerable. The usage
table never had one, so "this record was OPENED on project B" never was — and
the two cannot be joined to recover it, because there is no session identity
server-side (see the NoteUsageEvent docstring, which rules that out on
purpose).
That gap sits exactly on the question lessons exist to answer. A lesson's
whole claim is that it reaches a session on a project it was not written on:
"retrieved on a different project, at the moment it applies, AND opened". Surfacing cross-project: answerable. Opening:
answerable. Both at once: not, until this column.
WHICH PROJECT THIS IS, because the name is ambiguous and the wrong reading
makes the column useless. It is the project the READER was working in at the
moment of the event — NOT the project the record belongs to. The record's own
project is already on `notes.project_id`; copying it here would answer a
question nobody asked and quietly look like it answered this one.
NULLABLE, and not backfilled. Every existing row was written by a reader who
was somewhere, but naming it would be inventing a fact. Null means "not
reported", which is a different answer from a project id that might be wrong —
the same reasoning 0104 applied to its calibration stamps.
NULL IS ALSO THE ONGOING DEGRADED CASE, not only a historical one. A surfacing
arm always knows the project it searched; a getter only knows what its caller
passed. A pull with no project is still a pull and still counts toward "is this
dead weight" — it simply cannot answer "was it opened away from home".
THE INDEX carries `note_id` first, matching `ix_note_usage_note_event`: every
readout starts from a set of note ids and narrows, never from a project.
"""
import sqlalchemy as sa
from alembic import op
revision = "0105"
down_revision = "0104"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"note_usage_events",
sa.Column("project_id", sa.Integer(), nullable=True),
)
op.create_index(
"ix_note_usage_note_project",
"note_usage_events",
["note_id", "project_id"],
)
def downgrade() -> None:
op.drop_index("ix_note_usage_note_project", table_name="note_usage_events")
op.drop_column("note_usage_events", "project_id")
@@ -1,63 +0,0 @@
"""rule_usage_events carry an outcome, not just a read
Revision ID: 0106
Revises: 0105
Create Date: 2026-09-20
`rule_usage_events` can say a rule was SURFACED
and that it was PULLED. It cannot say what happened next, so these two
sessions leave identical telemetry:
- a rule surfaced, opened, and followed;
- a rule surfaced, opened, and silently ignored.
The second is the more urgent by a distance, and it is the one the readout
cannot name, and it is what an outcome column is for.
TWO NEW EVENT VALUES, AND NO CHECK MIGRATION. `event` was created in 0094 as
plain `sa.Text()` with no constraint — checked in the migration itself, not
assumed from the model — so `applied` and `departed` join `surfaced` and
`pulled` without a DROP/ADD pair. That pair is for CHECK-whitelisted columns
and this is not one; noted explicitly because the next reader will reach for
it here, and should be able to see in one place why it does not apply.
THE THIRD STATE IS DERIVED, AND THAT IS NOT A SHORTCUT. Read-and-silently-
unchanged is the absence of an outcome, and it has to be: an agent that knew
it was ignoring a rule would not be ignoring it. There is no honest way to ask
for that event, so nothing here tries. `applied` and `departed` are reported;
the third state is what is left over when a rule was pulled and neither
arrived. A schema that offered an `ignored` value would collect nothing and
read as though it had measured something — a statistic that cannot vary
being mistaken for a finding.
`detail` CARRIES THE WHY OF A DEPARTURE, and is the reason this is a column
rather than two more bare event strings. A departure without its reason is
indistinguishable from a miss when someone reads the table back, so the two
states the milestone wants to tell apart would collapse again one layer down.
Nullable because `applied` needs no argument — following a rule is the
unremarkable case, and demanding prose for it would make the cheap event
expensive and stop it being recorded at all.
NO NEW INDEX. Every outcome readout starts from a set of rule ids and narrows
by event, which is exactly `ix_rule_usage_rule_event` (rule_id, event) from
0094. Adding a `detail` index would serve no query anyone has — the column is
read, never filtered on.
"""
import sqlalchemy as sa
from alembic import op
revision = "0106"
down_revision = "0105"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"rule_usage_events",
sa.Column("detail", sa.Text(), nullable=True),
)
def downgrade() -> None:
op.drop_column("rule_usage_events", "detail")
@@ -1,66 +0,0 @@
"""system_embeddings — a charter becomes an ANSWER, not just a filter
Revision ID: 0107
Revises: 0106
Create Date: 2026-09-21
A System's `description` is a charter: several hundred words saying what
belongs in that area and what does not. It is the answer to "which part of
this codebase does X live in", and there was no semantic path to it.
`list_systems` enumerates and `search(system_id=…)` uses a System as a FILTER
over notes — so a System could narrow a search and could never be the answer
to one, and an agent asking where a record belonged had to read every charter
or guess.
The fourth sibling of note_embeddings (0067), rule_embeddings (0089) and
milestone_embeddings (0102), and for the same reason as the third: the row could be shared, the search cannot. "Where does this belong?"
is a different question from "what prior art is there?", and no note or rule
search can answer it, because a charter is not a note.
The vectors are DERIVED: nothing is backfilled here, the startup backfill
writes them.
"""
import sqlalchemy as sa
from alembic import op
revision = "0107"
down_revision = "0106"
branch_labels = None
depends_on = None
# Matches its three siblings — bge-small-en-v1.5, 384-dim.
_EMBEDDING_DIM = 384
def upgrade() -> None:
op.create_table(
"system_embeddings",
sa.Column(
"system_id", sa.Integer(),
sa.ForeignKey("systems.id", ondelete="CASCADE"), primary_key=True,
),
sa.Column("chunk_index", sa.Integer(), primary_key=True),
sa.Column("chunk_text", sa.Text(), nullable=False),
sa.Column("chunker_version", sa.Integer(), nullable=False),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False,
server_default=sa.text("now()"),
),
)
# Raw DDL for the vector column, as 0067, 0089 and 0102 do: the type comes
# from the pgvector extension, not SQLAlchemy's type system.
op.execute(
f"ALTER TABLE system_embeddings ADD COLUMN embedding vector({_EMBEDDING_DIM}) NOT NULL"
)
op.execute(
"""
CREATE INDEX ix_system_embeddings_embedding_hnsw
ON system_embeddings
USING hnsw (embedding vector_cosine_ops)
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_system_embeddings_embedding_hnsw")
op.drop_table("system_embeddings")
@@ -1,63 +0,0 @@
"""trigger_leaves_the_stored_title — a snippet's and lesson's title is its name
Revision ID: 0108
Revises: 0107
Create Date: 2026-09-23
Snippets and lessons stored their title as `subject — trigger`, because the
join is what makes these kinds rank on the situation they apply to and
the stored title WAS the embedded one. Every surface that shows a title then
showed the trigger too: menu lines, lists and search rows ran to 1,500–3,000
characters, and an injected repeat spent all of it again.
The trigger's home is `data`, and the join happens at embed time (`embeddings.document_title`). This revision
rewrites the rows already stored.
EXACT-SUFFIX, READ FROM THE ROW'S OWN MIRROR. A title is only rewritten when
it ends with `' — ' || <that row's trigger>`, so a subject that legitimately
contains an em dash is never cut, and a row whose title and mirror disagree —
hand-edited through the generic note door — is left alone. Such a row still
embeds correctly (`document_title` is idempotent) and merely shows its old
title; that is the right way round for a data migration to fail.
NO RE-EMBED, AND `updated_at` IS NOT TOUCHED. The embedded text is identical
before and after, so the vectors are already right. The startup backfill
re-embeds any row whose `updated_at` is newer than its vectors, and a raw
UPDATE that leaves `updated_at` alone is what keeps this from queueing the
whole snippet corpus for work that would produce the same numbers. There is
no database trigger on `updated_at`; it is set by the ORM only.
"""
from alembic import op
revision = "0108"
down_revision = "0107"
branch_labels = None
depends_on = None
# kind → the `data` key its trigger is mirrored under (embeddings._TRIGGER_DATA_KEYS).
_KINDS = (("snippet", "when_to_use"), ("lesson", "when_to_apply"))
_SEP = " — "
def upgrade() -> None:
for note_type, key in _KINDS:
op.execute(f"""
UPDATE notes
SET title = btrim(left(title, length(title) - length('{_SEP}' || (data->>'{key}'))))
WHERE note_type = '{note_type}'
AND coalesce(btrim(data->>'{key}'), '') <> ''
AND right(title, length('{_SEP}' || (data->>'{key}'))) = '{_SEP}' || (data->>'{key}')
""")
def downgrade() -> None:
# Recompose, on the same condition inverted: only where the trigger is not
# already on the end, so a downgrade run twice cannot double it.
for note_type, key in _KINDS:
op.execute(f"""
UPDATE notes
SET title = title || '{_SEP}' || (data->>'{key}')
WHERE note_type = '{note_type}'
AND coalesce(btrim(data->>'{key}'), '') <> ''
AND right(title, length('{_SEP}' || (data->>'{key}'))) <> '{_SEP}' || (data->>'{key}')
""")
@@ -1,42 +0,0 @@
"""embeddings_record_their_model — a vector says whose space it lives in
Revision ID: 0109
Revises: 0108
Create Date: 2026-09-23
Every embedding table stamps `chunker_version`, so a change to the document
shape is caught per row and re-embedded. None stamped the MODEL. The column is
`vector(384)` — a width, not an identity — so swapping bge-small for any other
384-dim model would write a second geometry beside the first with no error,
and search would go on ranking by cosines between the two, which mean nothing.
`embedding_model` is the other half of `calibration_stamp()`, stored per row on
all four tables. The startup backfill now re-embeds on either half moving.
THE BACKFILL LITERAL IS SAFE BECAUSE NO INSTALL HAS EVER CHANGED MODELS. The
name is hardcoded in `services/embeddings.py`, and every vector ever written was
written by it — so stamping every existing row with that name states a fact,
not a guess. It is frozen here rather than imported: a migration records what
was true when it ran, and a later model change must not rewrite history.
"""
from alembic import op
revision = "0109"
down_revision = "0108"
branch_labels = None
depends_on = None
_TABLES = ("note_embeddings", "rule_embeddings", "milestone_embeddings", "system_embeddings")
_MODEL = "BAAI/bge-small-en-v1.5"
def upgrade() -> None:
for table in _TABLES:
op.execute(f"ALTER TABLE {table} ADD COLUMN embedding_model text")
op.execute(f"UPDATE {table} SET embedding_model = '{_MODEL}'")
op.execute(f"ALTER TABLE {table} ALTER COLUMN embedding_model SET NOT NULL")
def downgrade() -> None:
for table in _TABLES:
op.execute(f"ALTER TABLE {table} DROP COLUMN embedding_model")
-38
View File
@@ -1,38 +0,0 @@
"""task_claims — a task can say a session is working it
Revision ID: 0110
Revises: 0109
Create Date: 2026-09-23
`status` is durable and nothing clears it, so `in_progress` cannot also mean
"someone is on this now". The claim is that second meaning, stored as WHO and
WHEN rather than as a flag, so it can be read as dead without anything having
cleared it (`services/task_claims.py` has the semantics).
Four nullable columns, no backfill: no existing row has ever been claimed, and
inventing a claim for one would assert attention nobody observed. No CHECK
enum is touched.
"""
import sqlalchemy as sa
from alembic import op
revision = "0110"
down_revision = "0109"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("notes", sa.Column(
"claimed_by", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL"), nullable=True,
))
op.add_column("notes", sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("notes", sa.Column("claim_touched_at", sa.DateTime(timezone=True), nullable=True))
op.add_column("notes", sa.Column("claim_session", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("notes", "claim_session")
op.drop_column("notes", "claim_touched_at")
op.drop_column("notes", "claimed_at")
op.drop_column("notes", "claimed_by")
@@ -1,53 +0,0 @@
"""lesson_rule_links — a lesson points at the rule it is an instance of
Revision ID: 0111
Revises: 0110
Create Date: 2026-10-01
One row per (lesson, rule) pair with a state: `suggested` while evidence
accumulates, `confirmed` or `rejected` once a judgment is made. Only a
confirmed link changes what surfaces; a rejected one is kept so the pair is not
proposed again. CASCADE on both ends — a link to a record that no longer exists
says nothing. No backfill: no lesson has ever been linked, and inventing a link
would assert a judgment nobody made.
"""
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision = "0111"
down_revision = "0110"
branch_labels = None
depends_on = None
# One place, so the CHECK and the model's LINK_STATES cannot drift (a new
# value later means DROP + ADD CONSTRAINT in the same migration).
_STATES = ("suggested", "confirmed", "rejected")
def upgrade() -> None:
op.create_table(
"lesson_rule_links",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column("lesson_id", sa.Integer(), sa.ForeignKey("notes.id", ondelete="CASCADE"), nullable=False),
sa.Column("rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), nullable=False),
sa.Column("state", sa.Text(), nullable=False, server_default="suggested"),
sa.Column("note", sa.Text(), nullable=True),
sa.Column("evidence", postgresql.JSONB(), nullable=True),
sa.Column("judged_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.UniqueConstraint("lesson_id", "rule_id", name="uq_lesson_rule_links_pair"),
)
op.create_check_constraint(
"ck_lesson_rule_links_state", "lesson_rule_links",
"state IN (" + ", ".join(f"'{s}'" for s in _STATES) + ")",
)
op.create_index("ix_lesson_rule_links_lesson_id", "lesson_rule_links", ["lesson_id"])
op.create_index("ix_lesson_rule_links_rule_id", "lesson_rule_links", ["rule_id"])
def downgrade() -> None:
op.drop_index("ix_lesson_rule_links_rule_id", table_name="lesson_rule_links")
op.drop_index("ix_lesson_rule_links_lesson_id", table_name="lesson_rule_links")
op.drop_constraint("ck_lesson_rule_links_state", "lesson_rule_links", type_="check")
op.drop_table("lesson_rule_links")
-32
View File
@@ -1,32 +0,0 @@
"""lesson_no_rule — "no rule fits" is a recorded answer
Revision ID: 0112
Revises: 0111
Create Date: 2026-10-01
One row per lesson that was judged to fall under no rule, with the reason. A
lesson with neither a confirmed link nor this row is UNJUDGED, which is what
`list_lessons(unjudged=true)` lists. CASCADE from the lesson. No backfill: no
lesson has been judged yet, and writing "no rule fits" for one nobody looked
at would assert a judgment nobody made.
"""
import sqlalchemy as sa
from alembic import op
revision = "0112"
down_revision = "0111"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"lesson_no_rule",
sa.Column("lesson_id", sa.Integer(), sa.ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True),
sa.Column("why", sa.Text(), nullable=False),
sa.Column("judged_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
def downgrade() -> None:
op.drop_table("lesson_no_rule")
@@ -1,33 +0,0 @@
"""system_path_patterns — a System names the files that are its area
Revision ID: 0113
Revises: 0112
Create Date: 2026-10-02
A JSONB list of globs relative to the repo root. NOT NULL with a `[]`
default, so an existing System reads as "no paths yet" rather than as NULL. No
backfill: which files are which area is a judgment, and the charters that name
directories in prose are not a mapping anyone confirmed.
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "0113"
down_revision = "0112"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"systems",
sa.Column(
"path_patterns", postgresql.JSONB(), nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
def downgrade() -> None:
op.drop_column("systems", "path_patterns")
@@ -1,45 +0,0 @@
"""system_usage_events — were an area's rulings read once its files were
touched?
Revision ID: 0114
Revises: 0113
Create Date: 2026-10-03
The usage twin for Systems, beside note_usage_events and rule_usage_events and
separate from both for the same reason they are separate from each other: a
System id restores through its own map. FK-free like its siblings; no CHECK on
`event`, like its siblings.
"""
import sqlalchemy as sa
from alembic import op
revision = "0114"
down_revision = "0113"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"system_usage_events",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False,
server_default=sa.text("now()"),
),
sa.Column("user_id", sa.BigInteger(), nullable=True),
sa.Column("system_id", sa.BigInteger(), nullable=False),
sa.Column("event", sa.Text(), nullable=False),
sa.Column("source", sa.Text(), nullable=False),
sa.Column("project_id", sa.BigInteger(), nullable=True),
)
op.create_index(
"ix_system_usage_system_event", "system_usage_events", ["system_id", "event"]
)
op.create_index("ix_system_usage_created_at", "system_usage_events", ["created_at"])
def downgrade() -> None:
op.drop_index("ix_system_usage_created_at", table_name="system_usage_events")
op.drop_index("ix_system_usage_system_event", table_name="system_usage_events")
op.drop_table("system_usage_events")
@@ -1,55 +0,0 @@
"""retrieval_judgments — a reviewer's verdict on each line of a logged menu
Revision ID: 0115
Revises: 0114
Create Date: 2026-10-03
Whether an injected line RELATED, which the usage tables cannot say: since
every line carries its matched passage, "not opened" no longer implies "not
relevant". FK-free like the telemetry tables beside it; no CHECK on `verdict`,
which the service validates.
"""
import sqlalchemy as sa
from alembic import op
revision = "0115"
down_revision = "0114"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"retrieval_judgments",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False,
server_default=sa.text("now()"),
),
sa.Column("user_id", sa.BigInteger(), nullable=True),
sa.Column("retrieval_log_id", sa.BigInteger(), nullable=False),
sa.Column("source", sa.Text(), nullable=False),
sa.Column("query", sa.Text(), nullable=True),
sa.Column("record_id", sa.BigInteger(), nullable=False),
sa.Column("rank", sa.Integer(), nullable=False),
sa.Column("score", sa.Float(), nullable=True),
sa.Column("within_budget", sa.Boolean(), nullable=False),
sa.Column("verdict", sa.Text(), nullable=False),
sa.Column("reason", sa.Text(), nullable=False),
sa.Column("opened_after", sa.Boolean(), nullable=True),
sa.UniqueConstraint(
"retrieval_log_id", "record_id", "user_id",
name="uq_retrieval_judgment_line",
),
)
op.create_index(
"ix_retrieval_judgments_source_created", "retrieval_judgments",
["source", "created_at"],
)
def downgrade() -> None:
op.drop_index(
"ix_retrieval_judgments_source_created", table_name="retrieval_judgments",
)
op.drop_table("retrieval_judgments")
-47
View File
@@ -1,47 +0,0 @@
"""moment_mappings — an install's own corrections to which actions reach which
moment
Revision ID: 0116
Revises: 0115
Create Date: 2026-10-05
The moments and their default actions are code; this table holds what an
install adds ("make ship" is a deliver here) and the defaults it switches off.
User-owned configuration, so it cascades with the user. No CHECK on `effect`,
which the service validates.
"""
import sqlalchemy as sa
from alembic import op
revision = "0116"
down_revision = "0115"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"moment_mappings",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False,
server_default=sa.text("now()"),
),
sa.Column(
"user_id", sa.Integer(),
sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False,
),
sa.Column("tool", sa.Text(), nullable=False),
sa.Column("match", sa.Text(), nullable=False, server_default=""),
sa.Column("moment", sa.Text(), nullable=False),
sa.Column("effect", sa.Text(), nullable=False, server_default="add"),
sa.Column("reason", sa.Text(), nullable=False, server_default=""),
sa.Column("actor", sa.Text(), nullable=False, server_default="model"),
sa.UniqueConstraint(
"user_id", "tool", "match", "moment", name="uq_moment_mapping",
),
)
def downgrade() -> None:
op.drop_table("moment_mappings")
-38
View File
@@ -1,38 +0,0 @@
"""rule_moments — the moments of work a rule arrives at
Revision ID: 0117
Revises: 0116
Create Date: 2026-10-05
A rule mounted on a moment is delivered whenever that moment happens, by
lookup rather than by score. The moment is a catalog name, validated by the
service; the catalog is code, so there is no table to key it to. Cascades
with the rule, like rule_systems beside it.
"""
import sqlalchemy as sa
from alembic import op
revision = "0117"
down_revision = "0116"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"rule_moments",
sa.Column(
"rule_id", sa.BigInteger(),
sa.ForeignKey("rules.id", ondelete="CASCADE"), primary_key=True,
),
sa.Column("moment", sa.Text(), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=True),
)
# The delivery read asks "which rules are on this moment?", the
# opposite direction from the primary key.
op.create_index("ix_rule_moments_moment", "rule_moments", ["moment"])
def downgrade() -> None:
op.drop_index("ix_rule_moments_moment", table_name="rule_moments")
op.drop_table("rule_moments")
@@ -1,60 +0,0 @@
"""rule_moment_judgments — whether a rule belongs on a moment, and who said so
Revision ID: 0118
Revises: 0117
Create Date: 2026-10-05
One row per (rule, moment) with a state: `suggested` while a proposal waits
for a judgment, `confirmed` or `rejected` once one is made. The mount itself
stays in rule_moments; this records the proposals that are not mounts yet and
the rejections that must stop them being proposed again. Moment '' is "no
moment fits this rule". CASCADE on the rule — a judgment about a rule that no
longer exists says nothing. No backfill: the mounts made before this table
were judgments too, but recording them now would invent who made them.
"""
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision = "0118"
down_revision = "0117"
branch_labels = None
depends_on = None
# One place each, so the CHECKs and the model's JUDGMENT_STATES / SOURCES
# cannot drift (a new value later means DROP + ADD CONSTRAINT in the same
# migration).
_STATES = ("suggested", "confirmed", "rejected")
_SOURCES = ("pass", "signal", "edit")
def upgrade() -> None:
op.create_table(
"rule_moment_judgments",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column("rule_id", sa.BigInteger(), sa.ForeignKey("rules.id", ondelete="CASCADE"), nullable=False),
sa.Column("moment", sa.Text(), nullable=False),
sa.Column("state", sa.Text(), nullable=False, server_default="suggested"),
sa.Column("source", sa.Text(), nullable=False, server_default="pass"),
sa.Column("note", sa.Text(), nullable=True),
sa.Column("evidence", postgresql.JSONB(), nullable=True),
sa.Column("judged_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.UniqueConstraint("rule_id", "moment", name="uq_rule_moment_judgments_pair"),
)
op.create_check_constraint(
"ck_rule_moment_judgments_state", "rule_moment_judgments",
"state IN (" + ", ".join(f"'{s}'" for s in _STATES) + ")",
)
op.create_check_constraint(
"ck_rule_moment_judgments_source", "rule_moment_judgments",
"source IN (" + ", ".join(f"'{s}'" for s in _SOURCES) + ")",
)
op.create_index("ix_rule_moment_judgments_rule_id", "rule_moment_judgments", ["rule_id"])
def downgrade() -> None:
op.drop_index("ix_rule_moment_judgments_rule_id", table_name="rule_moment_judgments")
op.drop_constraint("ck_rule_moment_judgments_source", "rule_moment_judgments", type_="check")
op.drop_constraint("ck_rule_moment_judgments_state", "rule_moment_judgments", type_="check")
op.drop_table("rule_moment_judgments")
@@ -1,36 +0,0 @@
"""rule_moment_judgments.misfire — a mount reported as arriving where it does
not apply
Revision ID: 0119
Revises: 0118
Create Date: 2026-10-05
The open-after-moment signal proposes a mount; nothing proposed taking one
off. This column is the evidence for that direction: each time a session says
a mounted rule arrived at a moment where it did not apply, the report is
counted here per distinct day, with its reason and the action that reached
the moment. At the bar it becomes an unmount proposal for the operator.
A column on the judgment row rather than a table of its own: a misfire is
always about a (rule, moment) pair that is mounted, and that pair already has
its row. Nullable and no backfill — nobody has reported a misfire yet.
"""
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision = "0119"
down_revision = "0118"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"rule_moment_judgments",
sa.Column("misfire", postgresql.JSONB(), nullable=True),
)
def downgrade() -> None:
op.drop_column("rule_moment_judgments", "misfire")
-288
View File
@@ -1,288 +0,0 @@
"""family canon — platforms, family ideas, the adoption ledger and its decision
log
Revision ID: 0120
Revises: 0119
Create Date: 2026-10-06
When one project solves something every project on the same platform will
meet, that solution becomes FAMILY CANON, and every other project on the
platform answers it: adopted, a variant (with a reason), exempt (with a
reason), or owed. These tables hold that.
- `platforms` — a GLOBAL catalog, the canonical_systems shape: no owner, slug
is the match key. Seeded with generic technology names only. Nothing here
names an app, a repo or a house convention: the catalog ships to every
install.
- `project_platforms` — which platforms a project is (declared / detected /
rejected). Membership is what makes "is this in family?" a lookup.
- `family_ideas` — a NOTE's family state. No new record type: any notes row
(a note, a snippet, a lesson) becomes an idea by gaining one of these.
- `family_idea_platforms` — the platforms an idea is for; the only scope
source. A linked rule topic takes its scope from the idea.
- `family_idea_references` — reference implementations (snippets).
- `family_adoptions` — one row per (project, idea): the project's answer.
- `family_decisions` — append-only log of every promotion and ledger change.
No backfill. No project has declared a platform and no note is an idea yet;
inventing either would assert a judgment nobody made. The seed rows are
written here verbatim rather than imported, so the migration keeps running
unchanged after any service-side list moves on.
"""
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
revision = "0120"
down_revision = "0119"
branch_labels = None
depends_on = None
# One place per whitelist, so each CHECK and the model's tuple cannot drift
# (a new value later means DROP + ADD CONSTRAINT in one migration).
_MEMBERSHIP = ("declared", "detected", "rejected")
_IDEA_STATUSES = ("candidate", "canon", "retired")
_ADOPTION_STATUSES = ("unassessed", "adopted", "variant", "exempt", "owed")
_REASONED = ("variant", "exempt")
_ACTIONS = ("propose", "promote", "revise", "retire", "assess", "undo")
_DECIDERS = ("agent", "operator", "system")
def _in(column: str, values: tuple[str, ...]) -> str:
return f"{column} IN (" + ", ".join(f"'{v}'" for v in values) + ")"
# (name, slug, description, markers). A marker with no slash matches a file's
# basename anywhere in a repo; one with a slash matches the repo-relative
# path. Empty markers = declare-only, for a platform that leaves no reliable
# file behind. Generic on purpose — an install adds its own.
_SEED = (
("Android app", "android-app",
"A native Android client: an APK or app bundle installed on phones and tablets.",
["AndroidManifest.xml"]),
("iOS app", "ios-app",
"A native iOS or iPadOS client built with Xcode.",
["project.pbxproj"]),
("Web frontend", "web-frontend",
"A browser client built with a frontend toolchain: single-page apps and server-rendered frontends.",
["vite.config.*", "svelte.config.*", "vue.config.*", "next.config.*",
"nuxt.config.*", "angular.json"]),
("Browser extension", "browser-extension",
"An extension installed into a web browser and distributed through its add-on store or by file.",
[]),
("Desktop app", "desktop-app",
"A packaged desktop application for Windows, macOS or Linux.",
["tauri.conf.json", "electron-builder.*"]),
("Container image", "container-image",
"Ships as an OCI/Docker image that a host pulls and runs.",
["Dockerfile", "Containerfile", "*.Dockerfile"]),
("Go", "go",
"Written in Go.",
["go.mod"]),
("Python", "python",
"Written in Python.",
["pyproject.toml", "setup.py", "requirements.txt"]),
("Rust", "rust",
"Written in Rust.",
["Cargo.toml"]),
("PostgreSQL", "postgresql",
"Keeps its data in PostgreSQL.",
[]),
("GitHub Actions", "github-actions",
"Verified and released by GitHub Actions workflows.",
[".github/workflows/*"]),
("Gitea / Forgejo Actions", "gitea-actions",
"Verified and released by Gitea or Forgejo Actions workflows.",
[".gitea/workflows/*", ".forgejo/workflows/*"]),
)
def upgrade() -> None:
platforms = op.create_table(
"platforms",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("name", sa.Text(), nullable=False),
sa.Column("slug", sa.Text(), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("markers", postgresql.JSONB(), nullable=False,
server_default=sa.text("'[]'::jsonb")),
sa.Column("order_index", sa.Integer(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("deleted_batch_id", sa.Text(), nullable=True),
)
# Unique among LIVE rows only (the canonical_systems convention).
op.create_index(
"uq_platforms_slug", "platforms", ["slug"],
unique=True, postgresql_where=sa.text("deleted_at IS NULL"),
)
op.bulk_insert(
platforms,
[
{"name": name, "slug": slug, "description": description,
"markers": markers, "order_index": index}
for index, (name, slug, description, markers) in enumerate(_SEED)
],
)
op.create_table(
"project_platforms",
sa.Column("project_id", sa.Integer(),
sa.ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True),
sa.Column("platform_id", sa.Integer(),
sa.ForeignKey("platforms.id", ondelete="CASCADE"), primary_key=True),
sa.Column("state", sa.Text(), nullable=False, server_default="declared"),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_check_constraint(
"ck_project_platforms_state", "project_platforms", _in("state", _MEMBERSHIP),
)
op.create_index("ix_project_platforms_platform_id", "project_platforms", ["platform_id"])
op.create_table(
"family_ideas",
sa.Column("note_id", sa.Integer(),
sa.ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True),
sa.Column("status", sa.Text(), nullable=False, server_default="candidate"),
sa.Column("applies_when", sa.Text(), nullable=True),
sa.Column("canon_version", sa.Integer(), nullable=False, server_default="1"),
# SET NULL: deleting a topic must not delete the standard it served.
sa.Column("topic_id", sa.BigInteger(),
sa.ForeignKey("rulebook_topics.id", ondelete="SET NULL"), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_check_constraint(
"ck_family_ideas_status", "family_ideas", _in("status", _IDEA_STATUSES),
)
# The first promotion criterion, held by the schema: an idea is not canon
# until it says, in platform terms, when it applies.
op.create_check_constraint(
"ck_family_ideas_canon_applies", "family_ideas",
"status <> 'canon' OR length(btrim(coalesce(applies_when, ''))) > 0",
)
op.create_check_constraint(
"ck_family_ideas_canon_version", "family_ideas", "canon_version >= 1",
)
op.create_index(
"uq_family_ideas_topic", "family_ideas", ["topic_id"],
unique=True, postgresql_where=sa.text("topic_id IS NOT NULL"),
)
op.create_index("ix_family_ideas_status", "family_ideas", ["status"])
op.create_table(
"family_idea_platforms",
sa.Column("note_id", sa.Integer(),
sa.ForeignKey("family_ideas.note_id", ondelete="CASCADE"), primary_key=True),
sa.Column("platform_id", sa.Integer(),
sa.ForeignKey("platforms.id", ondelete="CASCADE"), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_index("ix_family_idea_platforms_platform_id", "family_idea_platforms", ["platform_id"])
op.create_table(
"family_idea_references",
sa.Column("idea_id", sa.Integer(),
sa.ForeignKey("family_ideas.note_id", ondelete="CASCADE"), primary_key=True),
sa.Column("snippet_id", sa.Integer(),
sa.ForeignKey("notes.id", ondelete="CASCADE"), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_index("ix_family_idea_references_snippet_id", "family_idea_references", ["snippet_id"])
op.create_table(
"family_adoptions",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column("project_id", sa.Integer(),
sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False),
sa.Column("idea_id", sa.Integer(),
sa.ForeignKey("family_ideas.note_id", ondelete="CASCADE"), nullable=False),
sa.Column("status", sa.Text(), nullable=False, server_default="unassessed"),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("canon_version", sa.Integer(), nullable=True),
sa.Column("assessed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("decided_via", sa.Text(), nullable=True),
sa.Column("owed_task_id", sa.Integer(),
sa.ForeignKey("notes.id", ondelete="SET NULL"), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_check_constraint(
"ck_family_adoptions_status", "family_adoptions", _in("status", _ADOPTION_STATUSES),
)
# A departure is only a record if it says why.
op.create_check_constraint(
"ck_family_adoptions_reason", "family_adoptions",
"NOT (" + _in("status", _REASONED) + ") "
"OR length(btrim(coalesce(reason, ''))) > 0",
)
op.create_check_constraint(
"ck_family_adoptions_decided_via", "family_adoptions",
"decided_via IS NULL OR " + _in("decided_via", _DECIDERS),
)
op.create_index(
"uq_family_adoptions_pair", "family_adoptions", ["project_id", "idea_id"], unique=True,
)
op.create_index("ix_family_adoptions_project_id", "family_adoptions", ["project_id"])
op.create_index("ix_family_adoptions_idea_id", "family_adoptions", ["idea_id"])
op.create_table(
"family_decisions",
sa.Column("id", sa.BigInteger(), primary_key=True),
sa.Column("idea_id", sa.Integer(),
sa.ForeignKey("family_ideas.note_id", ondelete="CASCADE"), nullable=False),
sa.Column("project_id", sa.Integer(),
sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=True),
sa.Column("action", sa.Text(), nullable=False),
sa.Column("reason", sa.Text(), nullable=False),
sa.Column("before", postgresql.JSONB(), nullable=True),
sa.Column("after", postgresql.JSONB(), nullable=True),
sa.Column("evidence", postgresql.JSONB(), nullable=True),
sa.Column("precedent_ids", postgresql.JSONB(), nullable=False,
server_default=sa.text("'[]'::jsonb")),
sa.Column("decided_via", sa.Text(), nullable=False, server_default="agent"),
sa.Column("user_id", sa.Integer(),
sa.ForeignKey("users.id", ondelete="SET NULL"), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
)
op.create_check_constraint(
"ck_family_decisions_action", "family_decisions", _in("action", _ACTIONS),
)
op.create_check_constraint(
"ck_family_decisions_decided_via", "family_decisions", _in("decided_via", _DECIDERS),
)
# The reason IS the record — the agent decides with no approval step, so
# a decision that cannot say why is not one anybody can review or follow.
op.create_check_constraint(
"ck_family_decisions_reason", "family_decisions", "length(btrim(reason)) > 0",
)
op.create_index("ix_family_decisions_idea_id", "family_decisions", ["idea_id"])
op.create_index("ix_family_decisions_project_id", "family_decisions", ["project_id"])
def downgrade() -> None:
op.drop_index("ix_family_decisions_project_id", table_name="family_decisions")
op.drop_index("ix_family_decisions_idea_id", table_name="family_decisions")
op.drop_table("family_decisions")
op.drop_index("ix_family_adoptions_idea_id", table_name="family_adoptions")
op.drop_index("ix_family_adoptions_project_id", table_name="family_adoptions")
op.drop_index("uq_family_adoptions_pair", table_name="family_adoptions")
op.drop_table("family_adoptions")
op.drop_index("ix_family_idea_references_snippet_id", table_name="family_idea_references")
op.drop_table("family_idea_references")
op.drop_index("ix_family_idea_platforms_platform_id", table_name="family_idea_platforms")
op.drop_table("family_idea_platforms")
op.drop_index("ix_family_ideas_status", table_name="family_ideas")
op.drop_index("uq_family_ideas_topic", table_name="family_ideas")
op.drop_table("family_ideas")
op.drop_index("ix_project_platforms_platform_id", table_name="project_platforms")
op.drop_table("project_platforms")
op.drop_index("uq_platforms_slug", table_name="platforms")
op.drop_table("platforms")
@@ -1,58 +0,0 @@
"""retire the report_preference arm's rows
Revision ID: 0121
Revises: 0120
Create Date: 2026-10-09
The fixed-question arm that searched for completion-report preferences when a
task closed is gone: the reply shapes and the preferences mounted beside them
now arrive on the moments. Its rows go with it (there is no legacy to
preserve), because each one would otherwise mean something
different once the arm is no longer registered:
- `retrieval_logs` / `retrieval_judgments` — a source missing from the
registry reads as `unregistered_source`, whose advice is "add it to the
registry": the opposite of what happened.
- `rule_usage_events` — a source missing from `RANKED_SOURCES` reads as
ambient, so its surfacings would silently move into the other column of
every rule's pull-through.
- `retrieval_tuning_events` — the tuning history of a surface that no longer
exists.
- `settings` — the arm's floor and budget keys, which nothing reads.
COST IS BOUNDED BY AN INDEX, NOT BY THE TABLE. A migration runs at boot
against the live tables, which CI never has. `retrieval_logs`
and `retrieval_judgments` are indexed on `source`. `rule_usage_events` is not,
so its delete is also bounded by `created_at` (indexed): no row of the arm
predates `ARM_BORN`, and the scan covers weeks rather than the table's whole history.
Not reversible: the arm is not coming back, so neither are its rows.
"""
from alembic import op
revision = "0121"
down_revision = "0120"
branch_labels = None
depends_on = None
SOURCE = "report_preference"
# A safe floor under the arm's first row (it shipped after 2026-09-14).
ARM_BORN = "2026-09-01"
def upgrade() -> None:
op.execute(f"DELETE FROM retrieval_logs WHERE source = '{SOURCE}'")
op.execute(f"DELETE FROM retrieval_judgments WHERE source = '{SOURCE}'")
op.execute(
f"DELETE FROM rule_usage_events WHERE created_at >= '{ARM_BORN}' "
f"AND source = '{SOURCE}'"
)
op.execute(f"DELETE FROM retrieval_tuning_events WHERE surface = '{SOURCE}'")
op.execute(
"DELETE FROM settings WHERE key IN "
"('kb_reportpref_threshold', 'kb_reportpref_top_k')"
)
def downgrade() -> None:
pass
@@ -1,82 +0,0 @@
"""Seal forge-connection tokens at rest
Revision ID: 0122
Revises: 0121
Create Date: 2026-10-15
A forge token was stored as plain text, so a copy of the database (or of a
backup tool's snapshot of it) was enough to read every user's forge
credential. Tokens are now sealed the way the backup destination's S3 secret
is (services/secrets.py): Fernet under a key derived from SECRET_KEY, which
lives in the deployment's environment rather than in the data.
The column is renamed `token` -> `token_sealed` in the same step. The name
says what the bytes are, and every reader has to change with it, so a reader
missed here fails on a missing attribute instead of quietly sending
ciphertext to the forge as a token.
THE KEY DERIVATION IS FROZEN HERE, not imported. This revision must seal with
exactly v1's key forever; importing services.secrets would let a later edit
there change what an old migration writes. The unit lane checks that what
this produces is what `secrets.unseal` opens today.
Cost: one row per (user, forge host) — a handful per instance.
Downgrade opens each token back into plain text. A row sealed under a
different SECRET_KEY cannot be opened and comes back empty; its owner enters
the token again.
"""
import base64
import hashlib
import sqlalchemy as sa
from alembic import op
revision = "0122"
down_revision = "0121"
branch_labels = None
depends_on = None
_LABEL_V1 = b"scribe.secrets.v1\x00"
def _fernet():
from cryptography.fernet import Fernet
from grimoire.config import Config
digest = hashlib.sha256(_LABEL_V1 + Config.SECRET_KEY.encode()).digest()
return Fernet(base64.urlsafe_b64encode(digest))
def seal_v1(plain: str) -> str:
return _fernet().encrypt(plain.encode()).decode()
def upgrade() -> None:
op.alter_column("forge_connections", "token", new_column_name="token_sealed")
bind = op.get_bind()
rows = bind.execute(sa.text("SELECT id, token_sealed FROM forge_connections")).all()
for row_id, plain in rows:
bind.execute(
sa.text("UPDATE forge_connections SET token_sealed = :v WHERE id = :id"),
{"v": seal_v1(plain or ""), "id": row_id},
)
def downgrade() -> None:
from cryptography.fernet import InvalidToken
bind = op.get_bind()
fernet = _fernet()
rows = bind.execute(sa.text("SELECT id, token_sealed FROM forge_connections")).all()
for row_id, sealed in rows:
try:
plain = fernet.decrypt((sealed or "").encode()).decode()
except (InvalidToken, ValueError):
plain = ""
bind.execute(
sa.text("UPDATE forge_connections SET token_sealed = :v WHERE id = :id"),
{"v": plain, "id": row_id},
)
op.alter_column("forge_connections", "token_sealed", new_column_name="token")
@@ -1,64 +0,0 @@
"""drop user_profiles
Revision ID: 0123
Revises: 0122
Create Date: 2026-10-22
The Profile settings page asked for a display name, job title, expertise,
response style, tone, interests and a work schedule, and said an assistant
used them to personalise chat and a daily journal. Nothing in the app read
any of it: the page saved to this table and the backup carried it, and that
was all. The page, its API and the backup section are gone, and the table
goes with them.
Downgrade recreates the empty table, so the schema steps back cleanly; the
rows themselves do not come back.
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
revision = "0123"
down_revision = "0122"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_index("ix_user_profiles_user_id", table_name="user_profiles")
op.drop_table("user_profiles")
def downgrade() -> None:
op.create_table(
"user_profiles",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"user_id",
sa.Integer(),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
unique=True,
),
sa.Column("display_name", sa.Text(), nullable=True),
sa.Column("job_title", sa.Text(), nullable=True),
sa.Column("industry", sa.Text(), nullable=True),
sa.Column("expertise_level", sa.Text(), nullable=True),
sa.Column("response_style", sa.Text(), nullable=True),
sa.Column("tone", sa.Text(), nullable=True),
sa.Column("interests", ARRAY(sa.Text()), nullable=True),
sa.Column("work_schedule", JSONB(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
op.create_index("ix_user_profiles_user_id", "user_profiles", ["user_id"], unique=True)
@@ -1,45 +0,0 @@
"""skill names follow the plugin's name
Revision ID: 0124
Revises: 0123
Create Date: 2026-10-22
The plugin is named `grimoire`, so its skills reach a session as
`grimoire:<skill>` and stored processes as `grimoire-proc-<slug>`. A rule
mounted on one of those skills, a judgment about that mount, and an install's
own action mapping all store the skill's name, and a stored name the harness no
longer sends never matches again. This rewrites the plugin's earlier prefixes,
`scribe:` and `scribe-proc-`, to the current ones.
Cost: three small tables, each one row per mount, judgment or mapping.
"""
from alembic import op
revision = "0124"
down_revision = "0123"
branch_labels = None
depends_on = None
_MOMENT_TABLES = ("rule_moments", "rule_moment_judgments", "moment_mappings")
def _rename(old: str, new: str) -> None:
for table in _MOMENT_TABLES:
op.execute(
f"UPDATE {table} SET moment = regexp_replace(moment, "
f"'^skill\\.{old}(:|-proc-)', 'skill.{new}\\1') "
f"WHERE moment ~ '^skill\\.{old}(:|-proc-)'"
)
op.execute(
f'UPDATE moment_mappings SET "match" = regexp_replace("match", '
f"'skill={old}(:|-proc-)', 'skill={new}\\1', 'g') "
f"WHERE \"match\" ~ 'skill={old}(:|-proc-)'"
)
def upgrade() -> None:
_rename("scribe", "grimoire")
def downgrade() -> None:
_rename("grimoire", "scribe")
+15 -15
View File
@@ -1,4 +1,4 @@
# CI Requirements — FabledGrimoire
# CI Requirements — FabledScribe
> Spec lives in [`docs/process.md`](https://git.fabledsword.com/bvandeusen/CI-runner/src/branch/main/docs/process.md)
> in the CI-Runner repo.
@@ -16,10 +16,11 @@ real Postgres), build (docker buildx).
## Image deps used
- python 3.14
- node 24 (`npm ci` + `vue-tsc` in the typecheck job; the production
`Dockerfile` builds the frontend on `node:22-alpine`)
- ruff (lint job runs `ruff check src/ scripts/` with zero install overhead)
- uv (test + integration jobs run `uv sync --locked`)
- node 24 (used for `npm ci` + `vue-tsc` in the typecheck job, and as the
frontend builder stage inside the production `Dockerfile`)
- ruff (lint job runs `ruff check src/` with zero install overhead)
- uv (test + integration jobs run `uv sync --locked`; installed in the
image since the ci-python Dockerfile started pip-installing it)
- docker CLI + buildx (build job pushes the production image to the
Fabled-Git registry)
@@ -28,16 +29,15 @@ real Postgres), build (docker buildx).
Anything CI installs at job time that isn't in the image. Promotion
candidates if more than one project needs them.
- `shellcheck` — apt-installed in the **plugin** job, which lints the
Claude Code hook scripts and runs their fail-open smoke test. Grimoire is
its only consumer, so per `docs/process.md` it stays per-job. It is small
(~20 MB) and a promotion candidate once another project lints shell.
The hooks ship to users' machines, so they use only POSIX tools (awk, sed,
tr, od, cut, head, tail, grep, sort, date, printf) plus `git` and `curl`.
The smoke test runs on a bare image to keep it that way: a tool such as `jq`
is missing on macOS, slim Debian/Ubuntu and Alpine, and a hook that needed it
would give those machines nothing.
- `jq` + `shellcheck` — apt-installed in the **plugin** job, which lints
the four Claude Code hook scripts and runs their fail-open smoke test.
Per `docs/process.md`'s decision checkpoint, single-consumer deps stay
per-job until a second consumer wants them; Scribe is the only one so
far. Both are small (jq ~1 MB, shellcheck ~20 MB) and would be
promotion candidates the moment another project lints shell.
**jq is load-bearing for the smoke test specifically**: every hook
starts with `command -v jq || exit 0`, so without it the test passes
while exercising nothing.
## Notes
-35
View File
@@ -1,35 +0,0 @@
# Grimoire built from this checkout, for development:
# docker compose -f docker-compose.dev.yml up --build
services:
app:
build: .
ports:
- "5000:5000"
depends_on:
- db
restart: unless-stopped
volumes:
- app_data:/data
# Or a bind mount, for direct host access to the app's data:
# - ./data:/data
environment:
DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}"
# Optional: a SearXNG instance to list as the web-search backend in Integrations.
# SEARXNG_URL: "http://searxng:8080"
db:
# Postgres 17 with the vector extension Grimoire needs.
image: pgvector/pgvector:pg17
stop_grace_period: 120s
restart: unless-stopped
volumes:
- pgdata:/var/lib/postgresql/data
environment:
POSTGRES_USER: ${POSTGRES_USER:-grimoire}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire}
POSTGRES_DB: ${POSTGRES_DB:-grimoire}
volumes:
pgdata:
app_data:
+36 -15
View File
@@ -1,38 +1,59 @@
services:
app:
image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest
image: git.fabledsword.com/bvandeusen/fabledscribe:latest
environment:
DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${DB_PASSWORD}@db:5432/${POSTGRES_DB:-grimoire}"
# A Swarm can start the app on another node, away from the key kept in
# a node-local /data volume, so a stack sets its key itself (or mounts
# a Docker secret and sets SECRET_KEY_FILE to its path).
DATABASE_URL: "postgresql+asyncpg://scribe:${DB_PASSWORD}@db:5432/scribe"
SECRET_KEY: "${SECRET_KEY}"
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
TRUST_PROXY_HEADERS: "true"
SECURE_COOKIES: "true"
volumes:
- app_data:/data
networks:
- backend
- scribe_backend
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
deploy:
restart_policy:
condition: on-failure
max_attempts: 5
db:
# Postgres 17 with the vector extension Grimoire needs.
# pgvector image (Debian/glibc, PG17) — bundles the `vector` extension that
# migration 0067 enables. Moved off postgres:16-alpine via logical
# dump/restore (which doubles as the PG16->PG17 major upgrade); see the
# TRANSITION runbook in the PR.
image: pgvector/pgvector:pg17
stop_grace_period: 120s
volumes:
- pgdata:/var/lib/postgresql/data
environment:
POSTGRES_USER: ${POSTGRES_USER:-grimoire}
POSTGRES_USER: scribe
POSTGRES_PASSWORD: "${DB_PASSWORD}"
POSTGRES_DB: ${POSTGRES_DB:-grimoire}
POSTGRES_DB: scribe
networks:
- backend
- scribe_backend
# Lenient by design: a transient host exec/healthcheck stall (incident:
# runc setns failures -> "unhealthy" -> SIGKILL -> crash loop) must never
# escalate to killing the DB. Health here only gates app startup order.
healthcheck:
test: ["CMD-SHELL", "pg_isready -U scribe"]
interval: 30s
timeout: 10s
retries: 10
start_period: 180s
deploy:
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 0
window: 120s
volumes:
pgdata:
name: ${PGDATA_VOLUME:-grimoire_pgdata}
app_data:
networks:
backend:
scribe_backend:
driver: overlay
+59
View File
@@ -0,0 +1,59 @@
# Fabled Scribe — Quick Start
#
# No build required. Pulls the latest pre-built image from the registry.
#
# Usage:
# 1. Download this file
# 2. docker compose -f docker-compose.quickstart.yml up -d
# 3. Open http://localhost:5000 — the first account registered becomes admin
# 4. Go to Settings → MCP Access and connect Claude (Code or Desktop) via the
# bearer-token URL shown there.
#
# Set SECRET_KEY via environment variable or a .env file alongside this file:
# SECRET_KEY=your-random-secret-here
services:
app:
image: git.fabledsword.com/bvandeusen/fabledscribe:latest
ports:
- "5000:5000"
environment:
DATABASE_URL: "postgresql+asyncpg://scribe:scribe@db:5432/scribe"
SECRET_KEY: "${SECRET_KEY:-change-me-in-production}"
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
volumes:
- app_data:/data
depends_on:
db:
condition: service_healthy
restart: unless-stopped
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
db:
# pgvector image (PG17) — bundles the `vector` extension (migration 0067).
image: pgvector/pgvector:pg17
stop_grace_period: 120s
volumes:
- pgdata:/var/lib/postgresql/data
environment:
POSTGRES_USER: scribe
POSTGRES_PASSWORD: scribe
POSTGRES_DB: scribe
# Lenient by design: a transient host exec/healthcheck stall must never
# escalate to killing the DB. Health here only gates app startup order.
healthcheck:
test: ["CMD-SHELL", "pg_isready -U scribe"]
interval: 30s
timeout: 10s
retries: 10
start_period: 180s
restart: unless-stopped
volumes:
app_data:
pgdata:
+32 -25
View File
@@ -1,40 +1,47 @@
# Fabled Grimoire: the app and its database.
# docker compose up -d
# Then open http://localhost:5000. The first account registered becomes admin.
#
# Keep both volumes. app_data holds the instance's own SECRET_KEY, which seals
# saved credentials; pgdata holds everything else.
services:
app:
image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest
build: .
ports:
- "5000:5000"
environment:
DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}"
LOG_LEVEL: "${LOG_LEVEL:-INFO}"
# Behind an HTTPS reverse proxy:
# BASE_URL: "https://grimoire.example.com"
# SECURE_COOKIES: "true"
# TRUST_PROXY_HEADERS: "true"
depends_on:
db:
condition: service_healthy
volumes:
- app_data:/data
depends_on:
- db
restart: unless-stopped
# To use a bind mount instead (gives direct host access to all app data):
# - ./data:/data
environment:
DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-scribe}:${POSTGRES_PASSWORD:-scribe}@db:5432/${POSTGRES_DB:-scribe}"
SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}"
# Uncomment if you have a SearXNG instance you want to surface in the
# Integrations tab as a configured web-search backend:
# SEARXNG_URL: "http://searxng:8080"
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
db:
# Postgres 17 with the vector extension Grimoire needs.
image: pgvector/pgvector:pg17
image: postgres:16-alpine
stop_grace_period: 120s
restart: unless-stopped
volumes:
- pgdata:/var/lib/postgresql/data
environment:
POSTGRES_USER: ${POSTGRES_USER:-grimoire}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire}
POSTGRES_DB: ${POSTGRES_DB:-grimoire}
restart: unless-stopped
POSTGRES_USER: ${POSTGRES_USER:-scribe}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scribe}
POSTGRES_DB: ${POSTGRES_DB:-scribe}
# Lenient by design: a transient host exec/healthcheck stall must never
# escalate to killing the DB. Health here only gates app startup order.
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-scribe}"]
interval: 30s
timeout: 10s
retries: 10
start_period: 180s
volumes:
app_data:
pgdata:
app_data:
+271
View File
@@ -0,0 +1,271 @@
# Fable MCP — Design Spec
**Date:** 2026-03-23
**Status:** Approved
**Author:** bvandeusen + Claude
---
## Overview
A Python MCP (Model Context Protocol) server that lets Claude directly interface with a running Fabled Assistant instance. The goal is two-fold: Claude's stronger reasoning handles planning, documentation, and analysis while Fable remains the authoritative store for notes, tasks, projects, and milestones; and direct API access enables process improvement by letting Claude observe and operate on real data rather than working from descriptions.
The work is split into two sub-projects:
1. **Fable API Key Feature** — additions to the main `scribe` project to support bearer token authentication
2. **Fable MCP Server** — a new standalone Python package at `fable-mcp/` in the same repo root
A third sub-project (Forgejo MCP for CI/CD automation) is planned as a follow-on after the Fable MCP is working.
---
## Sub-project 1: Fable API Key Feature
### Motivation
All existing Fable routes use session-based authentication (`session["user_id"]`). The MCP server runs as a local process and cannot maintain a browser session, so a stateless bearer token mechanism is required. API keys are user-scoped and carry a read/write permission level.
### Database
New table `api_keys` via migration `0027_add_api_keys.py`:
- `down_revision = "0026"` (the briefing tables migration)
- `revision = "0027"`
| Column | Type | Notes |
|--------|------|-------|
| `id` | Integer PK | |
| `user_id` | Integer FK → users | CASCADE delete |
| `name` | Text | Human-readable label |
| `key_hash` | Text | SHA-256 of the full key, used for lookup |
| `key_prefix` | Text | First 8 chars (e.g. `fmcp_ab12`), shown in UI |
| `scope` | Text | `"read"` or `"write"` |
| `last_used_at` | Timestamp | Nullable, updated on each authenticated request |
| `created_at` | Timestamp | |
| `revoked_at` | Timestamp | Nullable — soft delete |
Full keys are generated as `fmcp_<32 random url-safe chars>`, returned once at creation, never stored. Only the SHA-256 hash is persisted.
### Auth Middleware (`auth.py`)
`_check_auth` is updated to check the `Authorization: Bearer <token>` header before falling back to session auth:
1. If header present: hash the token, look up in `api_keys` where `revoked_at IS NULL`
2. If found: update `last_used_at`, set `g.user` and `g.api_key`
3. Scope enforcement: if `g.api_key.scope == "read"` and `request.method` is not `GET`, return 403
4. If header absent: existing session logic runs unchanged
Session auth is untouched — no regression risk for the web UI.
**Admin routes and API keys:** Routes decorated with `admin_required` check `user.role == "admin"` after auth resolves. API keys authenticate as the key owner — so a write-scoped key for a non-admin user will fail admin-protected routes with 403 (role check, not scope check). This is intentional: API keys cannot elevate privilege beyond the user's role.
### Routes (`/api/api-keys`)
New blueprint `api_keys_bp`, registered in `app.py`:
- `GET /api/api-keys` — list caller's keys (prefix, name, scope, last_used_at, created_at — never the hash or full key)
- `POST /api/api-keys` — create key; body: `{name, scope}`. Returns full key in response **once only**
- `DELETE /api/api-keys/:id` — revoke by setting `revoked_at = now()`
### Settings UI
New "API Keys" tab in `SettingsView.vue` (added to `VALID_TABS`):
- Create form: name input + read/write radio/toggle + "Generate Key" button
- After creation: one-time modal displaying the full key with a copy button and a warning that it will not be shown again
- Keys table: columns for name, scope badge, prefix, last used, revoke button
- Revoke shows inline confirmation before calling DELETE
### New Search Endpoint
`GET /api/search?q=<query>&content_type=note|task|all&limit=N`
Calls the existing `semantic_search_notes()` service (already in `services/embeddings.py`). The `content_type` parameter maps to the service's `is_task` argument:
- `content_type=note` → `is_task=False`
- `content_type=task` → `is_task=True`
- `content_type=all` (default) → `is_task=None`
Returns:
```json
{
"results": [
{"id": 1, "title": "...", "body": "...", "is_task": false, "similarity": 0.87, "tags": [...]}
],
"total": 5
}
```
This endpoint is needed by the MCP's `search.py` tool module. It reuses existing infrastructure with no new ML work.
### Conversation Type: `"mcp"`
A new conversation type `"mcp"` is added alongside `"chat"` and `"briefing"`. This requires changes in two places in the main app:
**`services/chat.py`** — `create_conversation(user_id, title, model)` gains an optional `conversation_type: str = "chat"` parameter, passed through to the `Conversation` constructor.
**`routes/chat.py`** — `POST /api/chat/conversations` accepts an optional `conversation_type` body field (defaults to `"chat"`), passed to `create_conversation`. `GET /api/chat/conversations` continues to default-filter to `conversation_type="chat"` (excluding `"mcp"` and `"briefing"`); pass `?type=mcp` to retrieve MCP conversations.
**Retention:** `cleanup_old_conversations` in `routes/chat.py` currently deletes conversations regardless of type. It must be updated to exclude `conversation_type="mcp"` from the sweep, so MCP audit-trail conversations are not automatically pruned.
MCP conversations:
- Are created with a caller-supplied name (e.g., `"MCP Session 2026-03-23"`) or auto-named
- Are excluded from the default chat list
- Are accessible via `GET /api/chat/conversations?type=mcp` for inspection
- Appear in the Fable UI if explicitly navigated to, providing an audit trail of MCP-driven interactions
### Chat SSE Wire Format
The MCP's `chat.py` tool must consume the existing SSE stream. The relevant endpoints and event schema:
- **Create conversation:** `POST /api/chat/conversations` → `{id, title, ...}`
- **Post message + start generation:** `POST /api/chat/conversations/:id/messages` → `{message_id, ...}`; then connect to:
- **Stream:** `GET /api/chat/conversations/:id/stream` (SSE)
SSE event format (each line is `data: <json>`):
- `{"type": "token", "content": "..."}` — streaming token
- `{"type": "tool_call", "name": "...", "result": {...}}` — tool fired
- `{"type": "done", "content": "...", "tools_used": [...]}` — generation complete; this is the termination event
The MCP tool reads tokens until it receives `type: "done"`, then returns `response` (full content) and `tools_used` (list of tool names).
---
## Sub-project 2: Fable MCP Server
### Location
`fable-mcp/` at the repository root, alongside `src/`, `frontend/`, `alembic/`. It is **not** part of the main Docker build and has no import relationship with `scribe`. It will be extracted to its own Forgejo repo once stable.
### Package Structure
```
fable-mcp/
pyproject.toml # entry point: fable-mcp = "fable_mcp.server:main"
README.md
.env.example # FABLE_URL=http://localhost:8080, FABLE_API_KEY=fmcp_...
fable_mcp/
__init__.py
server.py # FastMCP instance, imports + registers all tool modules
client.py # FableClient: async httpx wrapper, env var config, FableAPIError
tools/
__init__.py
notes.py # list_notes, get_note, create_note, update_note, delete_note
tasks.py # list_tasks, get_task, create_task, update_task, delete_task,
# patch_task_status, add_task_log
projects.py # list_projects, get_project, create_project, update_project,
# delete_project, get_project_summary
milestones.py # list_milestones, get_milestone, create_milestone,
# update_milestone, delete_milestone
search.py # semantic_search — calls GET /api/search
chat.py # send_message — creates/continues conversation, returns response
```
### Dependencies
```toml
[project]
dependencies = [
"mcp[cli]>=1.0",
"httpx>=0.27",
"python-dotenv>=1.0",
]
```
### `client.py`
`FableClient` is an async context manager wrapping `httpx.AsyncClient`. It reads `FABLE_URL` and `FABLE_API_KEY` from environment variables (with `python-dotenv` fallback to `.env`). All methods are `async def` and return parsed dicts. A `FableAPIError(status_code, message)` exception is raised for any non-2xx response.
A module-level singleton `_client: FableClient` is initialized at server startup and shared across all tool modules.
### `server.py`
Uses `mcp[cli]`'s `FastMCP` class. Imports all tool modules, which register their tools via decorators against the shared `FastMCP` instance. Entry point `main()` calls `mcp.run()` (stdio transport).
### Tool Modules
Each module imports `_client` from `client.py` and defines tools as `async def` functions decorated with `@mcp.tool()`. Tool docstrings serve as the MCP tool descriptions visible to Claude.
**`notes.py`** tools:
- `list_notes(query, tags, project_id, limit, offset)`
- `get_note(note_id)`
- `create_note(title, body, tags, project_id)`
- `update_note(note_id, title, body, tags, project_id)`
- `delete_note(note_id)`
**`tasks.py`** tools:
- `list_tasks(query, project_id, milestone_id, status, priority, limit, offset)`
- `get_task(task_id)`
- `create_task(title, body, project_id, milestone_id, priority, status)`
- `update_task(task_id, title, body, project_id, milestone_id, priority, status)`
- `delete_task(task_id)`
- `patch_task_status(task_id, status)`
- `add_task_log(task_id, content)`
**`projects.py`** tools:
- `list_projects()`
- `get_project(project_id)` — includes milestone summary
- `create_project(title, description, goal, color)`
- `update_project(project_id, title, description, goal, status, color)`
- `delete_project(project_id)`
**`milestones.py`** tools:
- `list_milestones(project_id, status)`
- `get_milestone(project_id, milestone_id)`
- `create_milestone(project_id, title, description, order_index)`
- `update_milestone(project_id, milestone_id, title, description, status, order_index)`
- `delete_milestone(project_id, milestone_id)`
**`search.py`** tools:
- `semantic_search(query, content_type, limit)` — `content_type` is `"note"`, `"task"`, or `"all"` (avoids shadowing Python's `type` builtin). Calls `GET /api/search`, returns ranked results with similarity scores.
**`chat.py`** tools:
- `send_message(message, conversation_name)`:
- Looks up or creates a Fable conversation with the given name and `conversation_type="mcp"`
- Posts the message via `POST /api/chat/conversations/:id/messages`
- Consumes SSE stream from `GET /api/chat/conversations/:id/stream` until `type: "done"`
- Returns: `{response: str, tools_used: [str], conversation_id: int}`
- MCP conversations are excluded from the normal chat UI list
### Error Handling
`FableAPIError` is caught at each tool boundary and returned as a descriptive string rather than propagating as an exception. This ensures Claude receives a readable error message (e.g., `"Fable API error 403: Read-only key cannot perform write operations."`) rather than a stack trace. Network errors (`httpx.RequestError`) are similarly caught and surfaced as strings.
Scope enforcement lives entirely in Fable's auth middleware — the MCP does not duplicate it.
### Claude Code Registration
After `pip install -e fable-mcp/` (or `uv tool install ./fable-mcp`), add to `~/.claude/settings.json`:
```json
{
"mcpServers": {
"fable": {
"command": "fable-mcp",
"env": {
"FABLE_URL": "http://localhost:8080",
"FABLE_API_KEY": "fmcp_your_key_here"
}
}
}
}
```
Claude Code spawns the process over stdio automatically. No Docker, no daemon.
---
## Build & Repo Plan
1. Implement and test within `scribe/fable-mcp/`
2. Once stable, extract to a new Forgejo repo (`bvandeusen/fable-mcp`)
3. Forgejo MCP (Gitea MCP) added as a second MCP server to automate build/push/config workflows — separate spec when ready
---
## Out of Scope (v1)
- MCP resources (browsable URI tree) — tools cover all use cases in Claude Code
- Forgejo MCP — follow-on spec
- Rate limiting on API key endpoints
- Key expiry / TTL
- Per-resource scope (e.g., key restricted to one project)
File diff suppressed because it is too large Load Diff
+200
View File
@@ -0,0 +1,200 @@
# Speech-to-Speech (S2S) Design Spec
**Branch:** `feature/voice-s2s`
**Date:** 2026-03-29
**Status:** Approved for implementation
---
## Decisions
- **STT:** faster-whisper (in-process Python, model size configurable via `STT_MODEL` env var)
- **TTS:** Kokoro TTS (in-process Python, voice/speed/style configurable per-user)
- **Input mode:** Push-to-talk (Phase 1); VAD deferred
- **No browser STT/TTS fallbacks** — self-hosted only, data stays on-server
- **No Android app** — web only for this implementation
---
## New Env Vars (`config.py`)
| Var | Default | Description |
|---|---|---|
| `VOICE_ENABLED` | `false` | Feature flag — opt-in |
| `STT_BACKEND` | `faster-whisper` | Only supported value currently |
| `STT_MODEL` | `base.en` | `tiny.en` / `base.en` / `small.en` / `medium.en` |
| `TTS_BACKEND` | `kokoro` | Only supported value currently |
---
## Per-User Settings (stored in `settings` table)
| Key | Default | Description |
|---|---|---|
| `voice_tts_voice` | `af_heart` | Kokoro voice ID |
| `voice_tts_speed` | `1.0` | Speech rate, 0.7–1.3 |
| `voice_speech_style` | `conversational` | `conversational` / `concise` / `detailed` |
---
## New Backend Files
### `src/scribe/services/stt.py`
Lazy singleton `WhisperModel` loader. Public API:
- `load_stt_model()` — called at startup via `asyncio.create_task`
- `transcribe(audio_bytes, mime_type) -> str` — runs in `run_in_executor`; writes bytes to `NamedTemporaryFile`, returns concatenated segment text
- `stt_available() -> bool`
### `src/scribe/services/tts.py`
Lazy singleton `KPipeline` loader. Public API:
- `load_tts_model()` — called at startup
- `synthesise(text, voice, speed) -> bytes` — runs in `run_in_executor`; returns WAV bytes (24kHz, 16-bit mono)
- `list_voices() -> list[dict]` — returns static list of known Kokoro voice IDs + labels
- `tts_available() -> bool`
### `src/scribe/routes/voice.py`
Blueprint at `/api/voice`, all routes `@login_required`.
| Endpoint | Method | Description |
|---|---|---|
| `/api/voice/status` | GET | STT/TTS availability; `enabled` false if `VOICE_ENABLED=false` |
| `/api/voice/voices` | GET | List available Kokoro voices |
| `/api/voice/transcribe` | POST | multipart `audio` field → `{"transcript": "...", "duration_ms": 123}` |
| `/api/voice/synthesise` | POST | `{"text", "voice", "speed"}` → WAV bytes |
---
## Modified Backend Files
### `src/scribe/app.py`
- Register `voice_bp` blueprint
- In `startup()`: `asyncio.create_task(load_stt_model())` + `asyncio.create_task(load_tts_model())` when `VOICE_ENABLED`
### `src/scribe/config.py`
- Add 4 new env var attributes
- Add validation in `validate()`
### `src/scribe/services/llm.py`
- Add `voice_mode: bool = False` and `voice_speech_style: str = "conversational"` to `build_context()`
- When `voice_mode=True`, prepend: *"Respond naturally as if speaking aloud. No markdown, bullet points, headers, or code blocks. Complete sentences only."*
- Append style modifier based on `voice_speech_style`
### `src/scribe/services/generation_task.py`
- Add `voice_mode: bool = False` to `run_generation()`
- Read `voice_speech_style` from settings when voice_mode; pass both to `build_context()`
### `src/scribe/routes/chat.py`
- Allow `"voice"` in `conversation_type` whitelist
### `src/scribe/services/chat.py`
- Exclude `conversation_type == "voice"` from auto-cleanup retention
---
## New Frontend Files
### `frontend/src/composables/useVoiceRecorder.ts`
Wraps `MediaRecorder`. Exports: `recording`, `error`, `isSupported`, `startRecording()`, `stopRecording() -> Promise<Blob>`.
### `frontend/src/composables/useVoiceAudio.ts`
Wraps `AudioContext`. Exports: `playing`, `isSupported`, `play(blob)`, `stop()`.
### `frontend/src/components/VoiceOverlay.vue`
Floating PTT button (fixed bottom-right). Creates/reuses a `"voice"` conversation. Full flow: record → transcribe → send → stream → synthesise → play. Space bar hotkey (from `App.vue`). Mounted globally in `App.vue`.
---
## Modified Frontend Files
### `frontend/src/api/client.ts`
Add: `transcribeAudio(blob)`, `synthesiseSpeech(text, voice?, speed?)`, `getVoiceStatus()`, `getVoiceList()`
### `frontend/src/views/BriefingView.vue`
- "Listen" button: reads latest assistant message aloud via TTS
- Mic button in input bar: PTT → transcribe → auto-fill input → send
- Auto-TTS on assistant response when in listen mode
### `frontend/src/views/SettingsView.vue`
- New "Voice" tab: voice dropdown, speed slider, speech style radio
- Loads from `/api/settings`, saves via `PUT /api/settings`
### `frontend/src/App.vue`
- Mount `<VoiceOverlay />`
- Space bar → `"voice:ptt-toggle"` custom event
---
## Audio Format
| Direction | Format | Rationale |
|---|---|---|
| Browser → Server | WebM/Opus | Native `MediaRecorder` output; no re-encoding |
| Server → Browser | WAV (24kHz, 16-bit mono) | Kokoro native; no re-encoding; `decodeAudioData` compatible |
---
## Dependencies to Add (`pyproject.toml`)
```toml
[project.optional-dependencies]
voice = [
"faster-whisper>=1.0",
"kokoro>=0.9",
"soundfile>=0.12",
]
```
Install unconditionally in Docker (activated by `VOICE_ENABLED` at runtime):
```dockerfile
RUN pip install faster-whisper kokoro soundfile
```
---
## Database Migration
No schema changes required. `conversation_type` is unconstrained TEXT. Voice settings use existing key-value `settings` table. Optional no-op migration `0034_voice_conversation_type.py` for audit trail.
---
## Implementation Phases
### Phase 1 — Backend services + routes
1. Add env vars to `config.py`
2. Create `services/stt.py` (faster-whisper)
3. Create `services/tts.py` (Kokoro)
4. Create `routes/voice.py` (4 endpoints)
5. Wire model loading into `app.py` startup
6. Add `voice_mode` to `build_context()` + `run_generation()`
7. Allow `"voice"` conversation type in chat route + cleanup exclusion
### Phase 2 — BriefingView listen + voice follow-up
1. Create `useVoiceRecorder.ts`
2. Create `useVoiceAudio.ts`
3. Add voice API functions to `client.ts`
4. Add "Listen" button + mic button to `BriefingView.vue`
### Phase 3 — VoiceOverlay for general voice chat
1. Create `VoiceOverlay.vue`
2. Mount in `App.vue` + Space bar hotkey
### Phase 4 — Settings UI
1. Add "Voice" tab to `SettingsView.vue`
---
## Kokoro Voice Reference
| ID | Character |
|---|---|
| `af_heart` | American female, warm (recommended default) |
| `af_bella` | American female, expressive |
| `af_nicole` | American female, breathy/intimate |
| `af_sarah` | American female, clear |
| `af_sky` | American female, bright |
| `am_adam` | American male, neutral |
| `am_michael` | American male, deeper |
| `bf_emma` | British female |
| `bf_isabella` | British female, formal |
| `bm_george` | British male |
| `bm_lewis` | British male, casual |
+216
View File
@@ -0,0 +1,216 @@
# Agentic Briefing — Design Spec
**Date:** 2026-04-10
**Status:** Proposed
**Author:** bvandeusen + Claude
---
## Problem
The current briefing pipeline hallucinates calendar events, tasks, and news items that do not exist in the user's actual data. Observed examples from production:
- A morning briefing asserting "your dentist appointment is still in progress" when no such event existed
- An 8am check-in mentioning "a quick meeting at 10:30 AM" with no backing calendar entry
- A midday check-in inventing "a team huddle at 2:30 PM" and "the Q2 budget draft due by Friday"
- The model calling `search_images` in response to a clarifying question about a fabricated meeting
These are not bugs in data retrieval — the data gathering code (`_gather_internal`, `_gather_external`) returns correct values. They are a structural consequence of how the briefing context is assembled.
## Root cause — the "no receipt" problem
The current `run_compilation` pipeline does this:
1. Python code gathers data (tasks, events, weather, news) from the database
2. Python formats the data into a structured text blob as the user-role message: `TODAY'S EVENTS: ...`, `DUE TODAY: ...`, etc.
3. The LLM is called **once** with `[system_prompt, user_prompt]` and produces prose
4. Only the prose reply is written to the conversation. **The underlying data is never persisted in the conversation history.**
When the user later chats in the briefing conversation, the chat endpoint loads the full conversation history — which contains the model's *prose* from earlier but not the data that prose was derived from. The model's own prior output becomes the only source of "truth" available for follow-up questions. If that prose asserted a fact (real or hallucinated), the model has no way to distinguish it from ground truth when generating the next reply, and it will double down.
Compounding factors:
- **Empty sections are silently omitted.** If `calendar_events` is empty, the user prompt contains no `TODAY'S EVENTS:` line at all. The model has no explicit "zero events" signal — combined with an imperative system prompt ("note calendar events and tasks"), it interprets the silence as "I should mention some" and fabricates.
- **Scheduled slot injections append synthetic turns.** `run_slot_injection` writes a fake `[Midday briefing update]` user message and the assistant reply into the persistent conversation. By evening, the chat history contains three separate briefings, each potentially with errors, all treated as equal-weight context on follow-up.
- **The `search_images` tool is available during briefing chat**, with a negative-instruction description ("Not for factual questions"). Small and mid-sized models frequently ignore negative guidance in tool descriptions and call the tool anyway.
## Solution — agentic briefing (the receipt model)
Replace the one-shot synthesis with a tool-call loop. The briefing is no longer "a text blob synthesized from pre-gathered data." It is **a scheduled agent run**: the LLM is given a system prompt, a curated set of read-only data tools, and a trigger ("generate the morning briefing"). The model must call tools to see what exists. Every tool call and tool result becomes part of the conversation history, where it lives as a permanent, structured receipt.
### Why this fixes the hallucination
When the model calls `list_events(today) → []`, that empty array is now a persistent message in the conversation. On a follow-up question like "what meeting?", the chat endpoint loads the conversation and the model sees its own tool result from the morning showing no events. Answering "you had a meeting at 10:30" would require the model to directly contradict a tool result sitting two messages back — something LLMs are much more reliable at avoiding than contradicting their own prior prose.
In short: **the model cannot fabricate what has a visible receipt proving it does not exist.**
### What the model sees — before vs after
**Before** — the model sees one structured blob and produces prose. The blob is discarded.
```
[system] You are a personal assistant... weave together what matters...
[user] Date: 2026-04-10
WEATHER: Home — partly cloudy, 8–16°C
DUE TODAY: Fix briefing hallucinations
OVERDUE (2 tasks): ...
[assistant] Morning! Looks like a quiet day with two overdue items...
```
**After** — the model is given tools and iterates to ground truth.
```
[system] You are the user's personal assistant giving their morning briefing.
Use tools to see what's relevant. Only mention things you learned from
tool calls. If a tool returns nothing, say so honestly.
[user] Generate the morning briefing.
[assistant] tool_call: list_tasks(filter="due_today")
[tool] [{"id": 81, "title": "Fix briefing hallucinations", ...}]
[assistant] tool_call: list_tasks(filter="overdue")
[tool] [{"id": 42, ...}, {"id": 43, ...}]
[assistant] tool_call: list_events(today)
[tool] []
[assistant] tool_call: get_weather(location="home")
[tool] {"description": "partly cloudy", "temp_min": 8, "temp_max": 16}
[assistant] tool_call: get_rss_items(max=8)
[tool] [...]
[assistant] Morning! Partly cloudy today, 8 to 16 — nothing on the calendar
so it's a clean run at the desk. Two things to keep in mind...
```
The conversation now contains verifiable receipts: `list_events(today)` returned `[]`, and that result sits in context forever (until pruned). Follow-up questions operate against those receipts.
## Architecture
### Existing infrastructure (reused, not rebuilt)
The codebase already has the agentic primitives — they're used for regular chat:
- **`llm.py::stream_chat_with_tools`** — the streaming tool-use loop that talks to Ollama with a `tools` parameter and yields tool-call chunks
- **`generation_task.py::_stream_with_retry`** — wraps `stream_chat_with_tools` with retry-on-500 behavior for cold-model races
- **`tools.py`** — defines 40+ tool schemas and an execution dispatcher
The briefing bypasses all of this and calls a one-shot `_llm_synthesise` helper. The refactor is mainly "route briefings through the same pipeline regular chat already uses."
### New modules
**`briefing_tools.py`** — a small wrapper exposing a curated read-only subset of `tools.py` for briefing runs. This is an **explicit allowlist**, not a blocklist, so newly-added tools must be opted in:
| Tool | Purpose |
|---|---|
| `list_tasks` (with filter args) | See what's actionable today, overdue, high priority |
| `list_events` (today / upcoming) | Know what's on the calendar |
| `get_weather` | Current/forecast weather |
| `get_rss_items` | Pull news themes filtered by user preferences |
| `list_projects` | Understand project context |
| `search_projects` | Surface active project summaries |
| `list_notes` (recent) | Capture follow-ups from yesterday |
**Explicitly omitted** from the briefing tool set:
- `search_images`, `search_web`, `research_topic`, `read_article` — external search is not a briefing concern, and `search_images` is the source of the "Peter Kyle Science Secretary" image-search bug
- All `create_*`, `update_*`, `delete_*` — briefings are read-only; a scheduled background job must not decide to mutate the user's data on its own
- `set_rag_scope`, `calculate` — not relevant to briefing content
### New briefing function
**`briefing_pipeline.py::run_agentic_briefing(user_id, slot, model, conversation_id)`** replaces `run_compilation`'s body (and eventually `run_slot_injection`). Internally:
1. Build a slot-specific system prompt (see below)
2. Load the curated briefing tools from `briefing_tools.py`
3. Seed messages with `[system, user]` where the user message is a simple trigger like `"Generate the morning briefing."`
4. Enter a tool-call loop (max 8 iterations):
- Call `stream_chat_with_tools`
- If the model returns `tool_calls`, execute them via the existing dispatcher, append tool results, continue
- If the model returns a final assistant message with no pending tool calls, break
5. Return `(final_prose, full_message_list, metadata)`
The full message list is important: it's written to the conversation along with the final prose, so the tool-call receipts become part of the persistent record.
### Slot-specific system prompts
Compilation (full morning briefing):
```
You are the user's personal assistant giving their full morning briefing.
Use the tools available to see what's actually relevant today — tasks due,
overdue items, events on the calendar, weather, news themes, project state
— and weave it into a warm, natural-sounding summary.
Rules:
- Call tools to see the data. Never assert facts you didn't learn from a tool.
- If a tool returns nothing (no events today, no overdue tasks), say so
honestly. Don't fabricate items to fill space.
- Write flowing prose. No markdown, no headers, no bullet points.
- Aim for 6–10 sentences. Skip topics that have nothing interesting.
- Close on one or two concrete, actionable suggestions.
User profile (for tone and preferences):
{profile_body}
```
Check-ins (midday, afternoon):
```
You are the user's personal assistant giving a brief {slot} check-in.
Use tools to see what's changed since this morning. Focus on progress
and what's still unaddressed.
Rules:
- Call tools to see current state. Never assert facts without tool results.
- If nothing meaningful has changed, say so briefly — don't invent progress.
- 3–5 sentences, natural prose, no markdown.
```
### Conversation hygiene — removing the fake user messages
The current scheduler appends two fake messages on every slot injection:
```python
await post_message(conv.id, "user", f"[{slot.title()} briefing update]")
await post_message(conv.id, "assistant", text)
```
Under the agentic model, we drop the fake `"user"` message entirely. Slot updates become plain assistant messages tagged with `metadata.briefing_slot = "midday"`. The chat endpoint's message loader is updated to filter these when building the LLM context on follow-ups, so a user chatting in the briefing conversation doesn't see three earlier briefings smashed into their history. The UI continues to show them as visible timeline entries.
(This is the Option A filter from the earlier discussion — a small, surgical change compared to migrating briefings to a separate sidecar table. Sidecar storage remains a possible future step.)
### Ollama setup compatibility
The existing Ollama deployment uses non-parallel mode across two GPUs, with a concern about context duplication between cards. This is the correct setup for agentic briefings:
- Each tool-call iteration shares the same KV cache on the same GPU, so appending tool results is cheap
- There is no race where a second iteration could land on a different card with a different cache state
- The trade-off is serialization: a briefing in progress will block a concurrent user chat request until it finishes, but scheduled briefings are rare enough (4× per day) that this is acceptable
If GPU contention becomes a problem later, the right lever is pinning specific models to specific GPUs (e.g., background tasks on GPU 1, interactive models on GPU 0) — not enabling parallelism.
## Cost & trade-offs
- **Latency:** a briefing now makes 5–7 inference calls (one per tool-call decision plus the final prose) instead of 1. On a local Ollama with a 7B model, expect 15–40s per briefing vs the current 5–10s. Acceptable for a background job; if it becomes painful at the user-facing check-in slots, investigate letting the model batch independent tool calls in a single turn (Ollama supports this).
- **Model requirements:** the default model must be reliable at tool calling. `qwen2.5:7b`, `llama3.1:8b`, `mistral-small-3:24b`, and similar handle it well. Models in the ≤3B class typically fail — they either emit no tool calls and return empty prose, or hallucinate invalid tool arguments. If a user's `default_model` is too small, agentic mode should fall back to legacy mode with a log warning.
- **Context growth:** tool results bloat the conversation. At 8 tool calls per compilation × 4 slots per day, a daily briefing conversation can reach 20-30 KB of JSON-heavy history. Fine for a day; aged briefings should be archived/rolled up after 7 days.
- **Tool subset drift:** someone adds a new mutating tool to `tools.py` and forgets to update the briefing allowlist. Mitigated by the allowlist model — the default for new tools is "not exposed to briefings."
- **Infinite loop safety:** a buggy model could tool-call forever. Hard cap at 8 iterations, log a warning if hit, return whatever prose was last produced (or a fallback message).
## Migration path
Ship incrementally, each step independently reversible:
**PR 1 — Agentic compilation behind a feature flag.** Add `briefing_tools.py`, add `run_agentic_briefing`, add a per-user setting `briefing_mode: "legacy" | "agentic"` (default legacy). Route only the 4am compilation through the new path when the flag is set. Keep slot-injection on the legacy path. Enable the flag on the author's account first, validate output quality over several days, then flip the default for all users. No DB migration required — the setting lives in the existing `settings` table.
**PR 2 — Agentic slot injections + conversation hygiene.** Migrate midday/afternoon check-ins to the same pipeline. Remove the fake `[Midday briefing update]` user-role message; slot updates become plain assistant messages tagged with `metadata.briefing_slot`. Add a chat-message-loader filter that excludes slot-tagged messages from the LLM context on follow-ups (they remain visible in the UI).
**PR 3 — UI polish.** Collapsed tool-call status row in the briefing card ("✓ checked calendar · ✓ looked at tasks · ✓ pulled weather"), expanding to show tool results on click. Tool-result cards (weather, news, task list) rendered inline where useful.
**PR 4 (optional) — Sidecar storage for briefing snapshots.** If the chat-filter approach in PR 2 feels too hacky, migrate briefings out of the conversations table and into a dedicated `briefing_snapshots` table. Frontend renders them as pinned timeline cards separate from chat. Larger refactor; defer until PR 1–3 prove the approach works.
## Secondary win — tightening the main chat system prompt
The regular chat is already agentic (it uses `stream_chat_with_tools`), but its system prompt does not explicitly require the model to ground factual claims in tool results. The prompt discipline introduced for briefings — *"Never assert facts you didn't learn from a tool. If a tool returns nothing, say so honestly."* — is worth applying to the main chat system prompt in a follow-up PR. The mechanism already works; only the framing needs tightening.
## Out of scope
- The Android Flutter client's failure to render `search_images` tool-result cards. This is a separate rendering gap in the mobile app and does not affect the server-side fix. Tracked separately.
- Re-evaluating the Ollama parallelism setting. The current non-parallel config is correct for this work.
- Replacing the background model for title/summary/observation extraction. That model's role is unrelated to briefings.
+26 -42
View File
@@ -1,8 +1,8 @@
# API Keys and Grimoire MCP
# API Keys and Scribe MCP
## API Keys
API keys let external tools access your Grimoire data without a browser session. Each key is scoped to a single user — it can only access data that user owns or has been shared with them.
API keys let external tools access your Fable data without a browser session. Each key is scoped to a single user — it can only access data that user owns or has been shared with them.
### Scopes
@@ -15,51 +15,38 @@ Admin-level operations (log access, user management) require a `write`-scoped ke
### Creating a Key
1. Go to **Settings → MCP Access**
1. Go to **Settings → API Keys**
2. Enter a name (e.g. "Claude MCP", "Home Server")
3. Choose scope
4. Click **Generate Key**
5. Copy the key immediately — it is shown only once (the token is `fmcp_`-prefixed)
Paste the key into the `Authorization: Bearer <key>` header of your MCP client
config (see **Grimoire MCP Server** below).
config (see **Scribe MCP Server** below).
### Revoking a Key
Click **Revoke** next to the key in the keys table and confirm. Revoked keys are deleted immediately.
Click **Revoke** next to the key in the API Keys table and confirm. Revoked keys are deleted immediately.
---
## Grimoire MCP Server
## Scribe MCP Server
Grimoire exposes itself as a set of MCP tools that Claude (and other MCP clients)
Scribe exposes itself as a set of MCP tools that Claude (and other MCP clients)
can use to read and write your notes, tasks, projects, rulebooks, and more. The
server is **built into the app** — it is mounted as a streamable-HTTP endpoint
at **`/mcp`** on the running Grimoire instance (`src/grimoire/mcp/server.py`). There
at **`/mcp`** on the running Scribe instance (`src/scribe/mcp/server.py`). There
is nothing to install: no wheel, no separate package, no CLI. You connect a
client straight to the URL with a Bearer token.
### Authentication
Authenticate with an API key generated from **Settings → MCP Access** (see above),
Authenticate with an API key generated from **Settings → API Keys** (see above),
sent as `Authorization: Bearer fmcp_<key>`. A `read`-scoped key may call only the
read tools (`get_*`, `list_*`, `search`, `enter_project`, `retrieval_telemetry`);
any write/delete tool is rejected with `403`. The allow-list is explicit rather
than derived from the name — see `_READ_ONLY_TOOLS`, which is why the two reads
without a read-shaped name are spelled out here. A `write`-scoped key may call everything.
read tools (`get_*`, `list_*`, `search`, `enter_project`); any write/delete tool
is rejected with `403`. A `write`-scoped key may call everything.
### Claude Code: the plugin (recommended)
The plugin connects the MCP server and adds the hooks and skills that put Grimoire's records in front of Claude as it works. Install it from this repo:
```
/plugin marketplace add https://git.fabledsword.com/bvandeusen/FabledGrimoire.git
/plugin install grimoire@grimoire-plugin
```
It asks for your Grimoire base URL, an API key (kept in your OS keychain) and, optionally, a project id. Settings → MCP Access shows the exact command for your instance. [`plugin/README.md`](../plugin/README.md) describes each hook.
### Claude Code without the plugin
### Claude Code (Project-scoped)
Add a `.mcp.json` at the project root. The server `type` is `http` and the URL is
your instance's `/mcp` endpoint:
@@ -67,9 +54,9 @@ your instance's `/mcp` endpoint:
```json
{
"mcpServers": {
"grimoire": {
"scribe": {
"type": "http",
"url": "https://your-grimoire-instance.example.com/mcp",
"url": "https://your-scribe-instance.example.com/mcp",
"headers": {
"Authorization": "Bearer fmcp_your-api-key"
}
@@ -89,26 +76,23 @@ specific project at a dev instance or an admin key.
### Available Tools
There are well over a hundred tools and they change with the app, so the
registration in **`src/grimoire/mcp/tools/`** is the reference. By family:
The tool surface is large (~70 tools) and evolves with the app, so the live
registration in **`src/scribe/mcp/tools/`** is the source of truth rather than a
table here. The tools are grouped by family:
| Family | Examples | Purpose |
|--------|----------|---------|
| Notes | `create_note`, `get_note`, `update_note`, `create_records` | Knowledge, and several records that cite each other in one call |
| Tasks | `create_task`, `update_task`, `add_task_log` | Work, issues and spikes, and their logs |
| Projects / Milestones | `enter_project`, `start_planning`, `get_milestone` | Projects and their plans |
| Search / Recall | `search`, `what_might_apply`, `get_recent` | Recall by meaning, and the rules that may apply to an act |
| Systems | `create_system`, `list_system_records`, `map_system_to_canonical` | A project's areas |
| Rules | `create_rule`, `create_project_rule`, `relate_rules`, `rule_history`, `list_moments` | Rules, preferences, their relations and moments |
| Lessons | `create_lesson`, `judge_lesson_link` | Lessons and the rules they instance |
| Snippets / Shapes | `create_snippet`, `verify_snippet`, `classify_shapes`, `list_shapes` | Prior art and the shape ledger |
| Design systems | `resolve_design_system`, `get_design_system_stylesheet` | Tokens and guidance for UI work |
| Family canon | `list_family_ideas`, `assess_family_adoption` | Ideas shared across projects |
| Processes | `list_processes`, `get_process`, `create_process` | Stored procedures |
| Retrieval | `retrieval_telemetry`, `tune_retrieval` | What was surfaced, and the thresholds |
| Notes | `create_note`, `get_note`, `update_note`, `delete_note`, `list_notes` | Free-form knowledge |
| Tasks | `create_task`, `update_task`, `add_task_log`, `start_planning` | Actionable work + plans |
| Projects / Milestones | `enter_project`, `get_project`, `create_milestone`, … | Containers and outcomes |
| Search / Recall | `search`, `get_recent`, `list_tags` | Semantic + structured recall |
| Systems | `create_system`, `list_systems`, `list_system_records` | Reusable per-project subsystems/areas |
| Rulebooks | `list_always_on_rules`, `list_rules`, `create_rule`, `create_project_rule`, `subscribe_project_to_rulebook`, … | Engineering/workflow rules |
| Processes | `list_processes`, `get_process`, `create_process` | Saved prompts/workflows |
| Trash | `list_trash`, `restore`, `purge_trash` | Recoverable deletes |
| Admin | `get_app_logs` (write/admin key) | Diagnostics |
Server-level usage guidance — when to reach for each entity, the
recall-before-acting reflex, and the rulebook conventions — is delivered to the
client automatically via the MCP server's `instructions` block (defined in
`src/grimoire/mcp/server.py`).
`src/scribe/mcp/server.py`).
+12 -107
View File
@@ -52,9 +52,6 @@ endpoint at `/mcp`, not these REST routes.
| POST | `/api/notes/:id/versions/:vid/pin` | Pin a version |
| GET / PUT / DELETE | `/api/notes/:id/draft` | Unsaved-edit draft (restore across page loads) |
| GET | `/api/notes/graph` | Knowledge-graph nodes/edges |
| GET | `/api/notes/duplicates` | Near-duplicate notes or tasks, grouped |
| GET | `/api/notes/due-for-verification` | Notes that carry a check, oldest verification first |
| POST | `/api/notes/:id/verify` | Record that a note's check was run `{still_true}` |
## Tasks
@@ -79,9 +76,7 @@ endpoint at `/mcp`, not these REST routes.
| Method | Path | Description |
|--------|------|-------------|
| GET / POST | `/api/projects` | List (owned + shared) / create |
| GET / PATCH / DELETE | `/api/projects/:id` | Read (with `milestone_summary`, `inception`) / update / delete |
| POST | `/api/projects/:id/inception` | Record what the project inherits `{choices: {design_system_id, seed_systems}}` (owner-only; `POST /api/projects` accepts the same under `inception`) |
| GET | `/api/projects/:id/inception/defaults` | What binds if nobody decides — the inception card's payload |
| GET / PATCH / DELETE | `/api/projects/:id` | Read (with `milestone_summary`) / update / delete |
| GET | `/api/projects/:id/notes` | Notes + tasks in this project |
| GET / POST | `/api/projects/:id/milestones` | List / create milestones |
| GET / PATCH / DELETE | `/api/projects/:id/milestones/:mid` | Read / update / delete |
@@ -89,55 +84,7 @@ endpoint at `/mcp`, not these REST routes.
| GET / POST | `/api/projects/:id/systems` | List / create systems |
| GET / PATCH / DELETE | `/api/projects/:id/systems/:sid` | Read / update / delete |
| GET | `/api/projects/:id/systems/:sid/records` | Records linked to a system |
| PUT | `/api/systems/:sid/canonical` | Map a system to a shared area `{canonical_id}` (`null` unmaps) |
| GET / POST | `/api/canonical-systems` | List / create shared areas |
| PATCH | `/api/canonical-systems/:id` | Update a shared area |
| GET | `/api/projects/:id/issues` | Project issues |
| PUT | `/api/projects/:id/design-system` | Point the project at a design system `{design_system_id}` (`null` clears) |
| GET / PUT | `/api/projects/:id/platforms` | The project's platforms `{platforms: {slug: "declared" \| "rejected" \| null}}` |
| PUT | `/api/projects/:id/forge` | Pin the project to one forge connection, or clear it |
| GET / POST | `/api/platforms` | Platform catalogue / add a platform |
| PATCH | `/api/platforms/:id` | Update a platform |
## Snippets and lessons
| Method | Path | Description |
|--------|------|-------------|
| GET / POST | `/api/snippets` | List / create snippets |
| GET / PATCH / DELETE | `/api/snippets/:id` | Read / update / delete |
| POST | `/api/snippets/:id/verify` | Record a drift-check verdict `{status, detail, path}` |
| GET | `/api/snippets/duplicates` | Near-duplicate snippets, grouped into merge candidates |
| POST | `/api/snippets/:id/merge` | Merge source snippets into this one |
| POST | `/api/snippets/:id/unmerge` | Pull one source back out `{source_id}` |
| GET / POST | `/api/lessons` | List / create lessons |
| GET / PATCH / DELETE | `/api/lessons/:id` | Read / update / delete (to the trash) |
| GET | `/api/lessons/rule-candidates` | Rules a draft lesson resembles, before it is saved |
| GET | `/api/lessons/taught-by/:record_id` | Lessons drawn from one record |
| PUT | `/api/lessons/:id/rules/:rule_id` | Confirm or reject a lesson → rule link |
## Design systems
| Method | Path | Description |
|--------|------|-------------|
| GET / POST | `/api/design-systems` | List / create |
| GET | `/api/design-systems/starter-roles` | Starter token roles for the creation form |
| GET / PATCH / DELETE | `/api/design-systems/:id` | Read / update / delete |
| GET | `/api/design-systems/:id/resolved` | The effective token set, inheritance applied |
| GET | `/api/design-systems/:id/stylesheet` | The generated stylesheet |
| GET | `/api/design-systems/:id/snippet-check` | Snippets that disagree with the system's sheet |
| GET / POST | `/api/design-systems/:id/tokens` | This system's own tokens / add one |
| PATCH / DELETE | `/api/design-tokens/:id` | Update / delete a token |
## Family canon
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/family/ideas` | Family ideas |
| GET | `/api/family/ideas/:id` | One idea with its references |
| POST | `/api/family/ideas/:id/propose` · `/promote` · `/retire` | Move an idea through its lifecycle |
| GET | `/api/family/matrix` | Projects × canon ideas, with each project's answer |
| GET | `/api/family/decisions` | Promotion and retirement decisions |
| POST | `/api/family/decisions/:id/undo` | Undo a decision |
## Knowledge browse
@@ -166,30 +113,11 @@ endpoint at `/mcp`, not these REST routes.
| GET | `/api/rules` | List rules |
| POST | `/api/rulebook-topics/:tid/rules` | Add a rule to a topic |
| GET / PATCH / DELETE | `/api/rules/:id` | Read / update / delete a rule |
| POST | `/api/rules/:id/move` | Move a rule: `{topic_id}` makes it global, `{project_id}` makes it that project's |
| GET | `/api/projects/:id/rules` | A project's own rules, and the global rules tagged to its areas |
| POST | `/api/projects/:id/rulebook-subscriptions` | Subscribe a project to a rulebook |
| GET | `/api/projects/:id/rules` | Applicable rules for a project |
| POST | `/api/projects/:id/rules` | Create a project-scoped rule |
| GET | `/api/rules/:id/versions` | A rule's edit history, newest first |
| GET | `/api/rules/:id/versions/:vid` | One earlier version in full |
| POST | `/api/rules/:id/versions/:vid/restore` | Put a preference back to an earlier version (preferences only) |
| GET | `/api/rules/drift` | Preferences that have been rewritten, most recent first |
| POST | `/api/rules/:id/relations` | Relate this rule to another (`co_surfaces`, `overrides`, `elaborates`) |
| DELETE | `/api/rule-relations/:id` | Remove a relation |
| GET | `/api/rules-due-for-verification` | Rules that carry a check, oldest verification first |
| POST | `/api/rules/:id/verify` | Record that a rule's check was run `{still_true}` |
## Retrieval
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/retrieval/surfaces` | Every tunable surface with its threshold and budget |
| PUT | `/api/retrieval/surfaces/:surface` | Change one surface's setting, recorded with its reason |
| GET | `/api/retrieval/tuning-history` | Changes to retrieval on this install, newest first |
| GET | `/api/retrieval/moments` | The moments of work rules mount on, and the actions that reach each |
| POST / DELETE | `/api/retrieval/moments/mappings` | Map / unmap an action to a moment |
| GET | `/api/retrieval/moments/proposals` | Proposed rule → moment mounts awaiting a judgment |
| POST | `/api/retrieval/moments/proposals/judge` | Confirm or reject proposals |
| GET | `/api/retrieval/reply-shapes` | The default reply shapes and the moment each arrives at |
| POST / DELETE | `/api/projects/:id/suppressions/rules/:rid` | Suppress / unsuppress a rule |
| POST / DELETE | `/api/projects/:id/suppressions/topics/:tid` | Suppress / unsuppress a topic |
## Sharing
@@ -219,15 +147,13 @@ endpoint at `/mcp`, not these REST routes.
| POST | `/api/notifications/:id/read` | Mark one read |
| POST | `/api/notifications/read-all` | Mark all read |
## Settings
## Profile and Settings
| Method | Path | Description |
|--------|------|-------------|
| GET / PUT | `/api/profile` | Read / update the per-user profile |
| GET / PUT | `/api/settings` | Read / update key-value settings |
| GET | `/api/settings/search` | SearXNG configuration status |
| GET / POST | `/api/settings/forge-connections` | Your git forge connections / add one |
| PUT / DELETE | `/api/settings/forge-connections/:id` | Update / remove a connection |
| POST | `/api/settings/forge-connections/:id/test` | Check that the forge is reachable and accepts the token |
## API keys
@@ -238,22 +164,11 @@ endpoint at `/mcp`, not these REST routes.
## Plugin (Claude Code)
These serve the plugin's hooks.
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/plugin/context` | Session-start context: the active project and its state |
| GET | `/api/plugin/retrieve` | Records and rules matching the operator's prompt |
| GET | `/api/plugin/tool-rules` | Rules matching a tool call about to run |
| GET | `/api/plugin/moment-tools` | The tools whose calls can reach a moment |
| POST | `/api/plugin/moment` | The rules mounted on the moments a tool call reaches |
| POST | `/api/plugin/reply-rules` | Whether the reply ending a turn is held for one read |
| POST | `/api/plugin/rule-opened` | Records that a session opened a rule it was shown |
| GET | `/api/plugin/shape-check` | The end-of-turn question about what this turn wrote |
| GET | `/api/plugin/claim-session` · `/release-session` | Bind a session to its task claims / release them at session end |
| GET | `/api/plugin/context` | SessionStart context payload (rules + active-project) |
| GET | `/api/plugin/retrieve` | Title-first knowledge-injection candidates |
| GET | `/api/plugin/processes` | Stored Processes for skill-stub sync |
| GET | `/api/plugin/prior-art` | Write-path hint for the plugin hooks (params: `path`, `code`, `repo`, `shapes`, `exclude_ids`, `exclude_sync_ids`, `exclude_derive`); returns `context`, `note_ids`, `sync_note_ids`, `stamped`, `divergence`, `derive`, `derive_keys` |
| GET / POST | `/api/projects/<id>/coverage`, `…/coverage/refresh` | Shape-ledger accounting (`pattern_coverage` line, counts, `derive_groups` — css groups carry `consumers`, `derive_new`, `unused_css`, `divergence`, `recheck`) |
| GET / PUT | `/api/plugin/marketplace-url` | Read / set the plugin marketplace URL |
## Dashboard, Export, Trash, Users
@@ -280,22 +195,12 @@ These serve the plugin's hooks.
| GET / PUT | `/api/admin/base-url` | Get / set the public base URL |
| GET / PUT | `/api/admin/db-maintenance` (+ `/health`, POST `/run`) | VACUUM schedule, health, manual run |
| POST / GET / DELETE | `/api/admin/invitations` (+ `/:id`) | Create / list / revoke invite links |
| GET / PUT | `/api/admin/backups` | Scheduled backups: destination, schedule and recent runs |
| POST | `/api/admin/backups/check` | Ask the destination whether it would accept a backup |
| POST | `/api/admin/backups/run` | Start a backup now |
| GET / PUT | `/api/admin/forge-webhook` | The forge push-webhook settings |
## Webhooks
| Method | Path | Description |
|--------|------|-------------|
| POST | `/api/webhooks/forge` | Push events from a git forge, verified with `FORGE_WEBHOOK_SECRET` **(public)** |
## Grimoire MCP
## Scribe MCP
Claude clients connect to the built-in MCP server at `POST /mcp` (streamable HTTP,
Bearer auth with an `fmcp_` key), served by `src/grimoire/mcp/`. It is not a REST
Bearer auth with an `fmcp_` key), served by `src/scribe/mcp/`. It is not a REST
surface — it exposes the same data as typed tools (`create_note`, `create_task`,
`start_planning`, `search`, `enter_project`, …) with
`start_planning`, `search`, `enter_project`, `list_always_on_rules`, …) with
server-level usage guidance delivered in the MCP `instructions` block. See
[API Keys & MCP](api-keys-and-mcp.md).
+352 -67
View File
@@ -1,95 +1,380 @@
# Architecture
Grimoire is one app container and one PostgreSQL container. There is no language model in the stack: Claude works with Grimoire from outside, through the MCP endpoint and the Claude Code plugin, and people use the web UI.
## Stack
| Layer | Technology |
|-------|-----------|
| Frontend | Vue 3, TypeScript, Vite, Pinia, Vue Router; Tiptap for the editor |
| Backend | Python 3.14, Quart (async), served by Hypercorn |
| MCP | The `mcp` SDK's server, mounted in the same app at `/mcp` (streamable HTTP) |
| Database | PostgreSQL 17 with pgvector; SQLAlchemy 2 (async, asyncpg); Alembic migrations |
| Embeddings | fastembed running in-process (`BAAI/bge-small-en-v1.5`, 384 dimensions) |
| Plugin | Claude Code hooks (POSIX shell + awk), skills and a slash command, shipped from this repo |
| Layer | Technology | Notes |
|-------|-----------|-------|
| Frontend | Vue 3 + TypeScript + Vite + Pinia + Vue Router | SPA served from the same container as the API |
| Editor | Tiptap (ProseMirror) with custom slash-command extension | |
| Backend | Python 3.12, Quart (async ASGI) | Serves both API and built frontend static files |
| Database | PostgreSQL 16, SQLAlchemy 2.0 async, Alembic | asyncpg driver |
| LLM | Ollama (local) | Any OpenAI-compatible API also works |
| Search | SearXNG (optional, self-hosted) | Web search + image search |
| Push | Web Push / VAPID (pywebpush 2.x) | |
| Deployment | Docker Compose | Single-container app + separate DB + LLM service |
## How the pieces fit
## High-Level Component Diagram
```
Claude Code ──── plugin hooks ──── /api/plugin/* ┐
│ │
└────────── MCP tools ──────── /mcp ├── Quart app ── PostgreSQL + pgvector
│ │
Browser ──────── Vue SPA ────────── /api/* ┘ └── in-process jobs and embedder
┌─────────────────────────────────────────────┐
│ Docker Compose │
│ │
│ ┌──────────────────────┐ ┌────────────┐ │
│ │ scribe │ │ ollama │ │
│ │ ┌────────────────┐ │ │ │ │
│ │ │ Quart Server │ │ │ LLM API │ │
│ │ │ ┌──────────┐ │ │ │ │ │
│ │ │ │ Vue SPA │ │ │ └────────────┘ │
│ │ │ │ (static) │ │ │ ▲ │
│ │ │ └──────────┘ │ │ │ │
│ │ │ ┌──────────┐ │ │ HTTP/REST │
│ │ │ │ /api/* │──┼──┼─────────┘ │
│ │ │ └──────────┘ │ │ │
│ │ │ │ │ │ ┌────────────┐ │
│ │ │ ▼ │ │ │ PostgreSQL │ │
│ │ │ ┌──────────┐ │ │ │ 16 │ │
│ │ │ │ asyncpg │──┼──┼──▶ │ │
│ │ │ └──────────┘ │ │ └────────────┘ │
│ │ └────────────────┘ │ │
│ └──────────────────────┘ │
└─────────────────────────────────────────────┘
```
- **The web UI** is a Vue single-page app, built into the image and served by Quart. Every unknown path that isn't under `/api/` falls back to `index.html`; `/api/*` always answers JSON.
- **The REST API** (`src/grimoire/routes/`) backs the UI, one blueprint per resource.
- **The MCP server** (`src/grimoire/mcp/`) is how Claude reads and writes records. Each module in `mcp/tools/` registers its tools, and the server's instructions and each tool's docstring are the contract an agent reads. Requests authenticate with an API key as a Bearer token.
- **The plugin** (`plugin/`) runs in the user's Claude Code. Its hooks call `/api/plugin/*` to put the right context in front of Claude at the right moment: the project at session start, matching rules and prior art before a prompt or a tool call, the rules mounted on a moment of work, and a check on the reply that ends a turn. Its skills describe the working practices. See [API Keys & MCP](api-keys-and-mcp.md) to install it.
- **Services** (`src/grimoire/services/`) hold the logic both doors share. Routes and MCP tools stay thin: parse the request, check access, call a service, return a plain dict.
## Project Structure
## Data model
```
scribe/
├── docker-compose.yml # Development stack
├── docker-compose.prod.yml # Production stack (Docker Swarm)
├── Dockerfile # Multi-stage build (Node → Python)
├── alembic/ # Database migrations
│ └── versions/ # Migration files (idempotent raw SQL)
├── src/scribe/
│ ├── app.py # Quart app factory + blueprint registration
│ ├── config.py # Config class (reads env vars)
│ ├── auth.py # login_required decorator, session checks
│ ├── models/ # SQLAlchemy models
│ ├── mcp/ # In-app MCP server (FastMCP, mounted at /mcp)
│ │ ├── server.py # FastMCP instance + instructions + Quart mount
│ │ └── tools/ # Tool modules (notes, tasks, projects, rulebooks, …)
│ ├── routes/ # API blueprints (one file per resource)
│ ├── services/ # Business logic (access, llm, tools, sharing, …)
│ └── static/ # Built Vue SPA (generated at Docker build time)
└── frontend/
└── src/
├── views/ # Page-level Vue components
├── components/ # Reusable UI components
├── composables/ # Vue composables (autosave, shortcuts, …)
├── stores/ # Pinia stores (auth, chat, notes, notifications, …)
└── api/ # Typed API client (client.ts)
```
Most records are rows in one `notes` table, told apart by kind:
## Key Design Decisions
- **Notes**, and the specialised kinds that are notes underneath: **snippets** (recorded prior art, with a canonical code location), **lessons** (transferable insights, linked to the rules they instance) and **processes** (stored procedures, synced to Claude Code as skills).
- **Tasks**: a note with a status. A task's kind is `work`, `issue` or `spike`. Tasks carry work logs, sub-tasks, recurrence and a session claim.
**Single container for frontend + API.** Quart serves the Vue.js production build as static files and exposes the REST API under `/api/`. The SPA is built by Vite during the Docker image build.
Around them:
**SPA routing via 404 handler.** `app.py` uses `@app.errorhandler(404)` (not a catch-all route) to serve static files or fall back to `index.html`. API routes (`/api/*`) always return JSON 404. This avoids a catch-all `/<path:path>` route intercepting API GETs.
- **Projects** group work and carry a goal. **Milestones** are plans: a design body plus ordered step tasks.
- **Systems** are a project's named areas; records are tagged with them, and each maps to a canonical system shared across projects.
- **Rulebooks → topics → rules.** A rule in a rulebook is global; a rule on a project applies there only. Rules have a trigger (`when_to_apply`), versions, relations to other rules, and can be mounted on **moments** of work (`work.change`, `reply.report`, ...), which plugin actions reach through moment mappings.
- **Family canon**: ideas shared across projects on the same platform, and each project's answer to them (adopted, variant, owed).
- **Design systems**: tokens and guidance, with inheritance; the stylesheet is generated from them.
- **The shape ledger**: code shapes extracted from bound repos and their classification against snippets.
- **Embeddings** for notes (chunked), rules, milestones and systems, in pgvector columns.
- **Telemetry**: retrieval logs, note and rule usage, and the tuning history of retrieval thresholds.
- **Users, groups, shares and API keys.** Access is decided in the app (`services/access.py`): owners, direct shares and group shares, with read or write permission.
**Unified note/task model.** A task is just a note with task attributes enabled. `status IS NOT NULL` means it's a task. "Convert to task" sets `status='todo'`; "convert to note" clears `status`, `priority`, `due_date`. No separate table, no cascade complexity.
Deleted records go to the trash first and can be restored until retention purges them.
**First-class tag column.** Tags live in a `tags ARRAY[text]` column and are explicitly set by the client — not auto-extracted from body text. Hierarchical tags (`project/webapp`) supported via SQL `unnest + LIKE` prefix matching.
## Retrieval
**Background generation architecture.** LLM streaming runs in a detached `asyncio.Task` that writes into an in-memory `GenerationBuffer`. SSE clients tail the buffer and can reconnect mid-stream without data loss. Buffer has a `cancel_event` for user-initiated stop. Completed buffers are cleaned up after 60s grace period. Periodic DB flushes every 5s preserve partial content. Both chat and AI Assist use this architecture.
Search and the plugin's context injection run on the same pipeline (`services/retrieval_pipeline.py`): embed the query, rank stored chunks by cosine similarity in pgvector, keep what clears the surface's threshold, and log what was shown. Each surface (MCP search, auto-inject, the pre-tool rule arm, the write-path prior-art arm, ...) is a spec over those stages with its own thresholds and budget. The thresholds are settings an admin can tune, and every change is recorded with its reason. When the embedder is unavailable, search falls back to keyword matching.
**SSE over WebSockets for LLM streaming.** SSE clients connect via `GET /api/chat/conversations/:id/generation/stream` with `Last-Event-ID` reconnection support. Frontend uses `fetch()` + `ReadableStream`.
The embedding model downloads into `FASTEMBED_CACHE_DIR` (default `/data/fastembed-cache`) on first use, so the `/data` volume keeps it across restarts.
**Context building is server-side.** Backend fetches URL content and searches notes. Frontend sends the message text + optional context note IDs. `build_context()` returns `(messages, context_meta)`; metadata includes auto-found note IDs/titles sent to frontend via a `context` SSE event before streaming begins.
## Background work
**No blocking long-running operations.** Any slow operation (model pulls, LLM calls, URL fetching) must never block app startup or freeze the UI. Backend uses SSE streaming for incremental responses. Model pulls stream NDJSON progress to the frontend.
All in-process; there is no separate worker container.
**SSRF protection.** `services/llm.py` blocks requests to loopback, private, link-local, reserved, and multicast addresses before fetching. `follow_redirects=False` prevents redirect-based bypasses.
- At startup the app records any shipped default that moved in this release, starts its schedulers, and only then backfills missing embeddings, so the backfill never competes with startup for the database.
- Recurring tasks spawn every 15 minutes. Daily jobs: version auto-pinning (03:00 UTC), trash purge (03:30 UTC), targeted `VACUUM ANALYZE` (04:00 UTC by default). Scheduled backups check hourly whether one is due ([Backups](backups.md)).
- Log, notification and auth-token retention run on their own loops.
- A diagnostics heartbeat writes to `/data/diagnostics`.
**Session cookie security.** `HttpOnly` and `SameSite=Lax` always set. `Secure` flag controlled by `SECURE_COOKIES` env var.
Every database wait on startup has a deadline. Migrations wait for the database to accept connections (up to five minutes) before they run, so the app starts cleanly whatever order the containers come up in.
**Rate limiting.** In-memory sliding-window rate limiter (`rate_limit.py`) applied to auth endpoints: login (10/60s), register (5/300s), forgot-password (5/300s), reset-password (10/60s). Keys are per-IP. `TRUST_PROXY_HEADERS` env var enables `X-Forwarded-For` / `X-Real-IP` when behind a reverse proxy.
**Idempotent migrations.** All Alembic migrations use raw SQL with `CREATE TABLE IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and `DO $$ BEGIN CREATE TYPE ... EXCEPTION WHEN duplicate_object` to allow safe re-runs.
## Data Model
### Users
| Column | Type | Notes |
|--------|------|-------|
| `id` | int PK | |
| `username` | text UNIQUE NOT NULL | |
| `email` | text nullable | |
| `password_hash` | text nullable | NULL for OAuth-only accounts |
| `oauth_sub` | text UNIQUE nullable | OIDC subject identifier |
| `role` | text NOT NULL DEFAULT 'user' | First user auto-assigned 'admin' |
| `session_version` | int NOT NULL DEFAULT 1 | Bumped on password change to evict sessions |
| `created_at` | timestamptz | |
### Notes (unified — includes tasks)
| Column | Type | Notes |
|--------|------|-------|
| `id` | int PK | |
| `title` | text | |
| `body` | text | Markdown |
| `tags` | ARRAY[text] | GIN indexed; explicitly set by client |
| `parent_id` | int FK self nullable | Sub-tasks / sub-notes |
| `user_id` | int FK users nullable | CASCADE |
| `project_id` | int FK projects nullable | |
| `milestone_id` | int FK milestones nullable | |
| `status` | text nullable | `todo`/`in_progress`/`done` — non-null = task |
| `priority` | text nullable | `none`/`low`/`medium`/`high` |
| `due_date` | date nullable | |
| `created_at`, `updated_at` | timestamptz | |
Indexes: GIN on `tags`, B-tree on `status`, B-tree on `title`.
### Settings
Composite PK `(user_id, key)`. Per-user key-value store. CRUD via `services/settings.py`. Used for: `default_model`, `assistant_name`, `journal_config` (JSON: locations, temp_unit, prep schedule), `user_timezone`, `voice_*`, etc.
### Conversations / Messages
`conversations`: `id`, `title`, `model`, `user_id`, `conversation_type` (`chat`/`journal`/`mcp`), `day_date` (YYYY-MM-DD for journal conversations), `rag_project_id` (nullable int — RAG scope: NULL=orphan-only, -1=all, positive=project), `created_at`, `updated_at`.
`messages`: `id`, `conversation_id` FK CASCADE, `role` (`user`/`assistant`), `content`, `status` (`done`/`generating`), `created_at`.
Title auto-generated by LLM on first exchange, re-generated every 10th message.
### Projects / Milestones
`projects`: `id`, `user_id`, `title`, `description`, `goal`, `status` (`active`/`completed`/`archived`), `color`, `auto_summary` (nullable text — LLM-generated summary for `search_projects` scoring), `summary_updated_at` (nullable timestamptz), timestamps.
`milestones`: `id`, `user_id`, `project_id` FK CASCADE, `title`, `description`, `status`, `order_index`, timestamps.
### Sharing & Access
`project_shares`, `note_shares`: each has `shared_with_user_id` OR `shared_with_group_id` (exclusive), `permission` (`viewer`/`editor`/`admin`), `invited_by`.
`groups`, `group_memberships`: platform-wide groups with `member`/`owner` roles.
Permission resolution is centralised in `services/access.py`. `get_project_permission(uid, project_id)` checks ownership → direct share → group-based share → note→project inheritance, returning the highest applicable permission.
### Journal-Related Tables
`weather_cache`: per-user, per-`location_key` cache. Columns: `user_id`, `location_key` (`home`/`work`/etc.), `location_label`, `forecast_json` (Open-Meteo response), `previous_json` (last forecast, used to detect changes), `fetched_at`. Lat/lon are *not* stored on the cache row — they live in the user's `journal_config.locations.{home|work}` setting and are used at refresh time.
### API Keys
`api_keys`: `id`, `user_id` FK CASCADE, `prefix` (first 8 chars, displayed in UI), `key_hash` (SHA-256 of full key — full key never stored), `name`, `scope` (`read`/`write`), `created_at`, `last_used_at`.
### App Logs
`category` (`audit`/`usage`/`error`), `user_id` FK nullable (SET NULL on delete), `username` (denormalised), `action`, `endpoint`, `method`, `status_code`, `duration_ms`, `error_type`, `error_message`, `traceback`, `details` JSONB.
## Detailed File Reference
### Backend (`src/scribe/`)
| File | Responsibility |
|------|---------------|
| `app.py` | Quart app factory; SPA via 404 handler; JSON 404/500 for API; request logging; security headers in `after_request` |
| `auth.py` | `login_required`, `admin_required`, `get_current_user_id` — shared `_check_auth()` helper; accepts session cookie or `Authorization: Bearer <key>` |
| `config.py` | All config from env vars + Docker secret file support (`_read_secret`); `SECURE_COOKIES`, `TRUST_PROXY_HEADERS`, `OLLAMA_NUM_CTX`; `oidc_enabled()`, `searxng_enabled()` classmethods |
| `rate_limit.py` | In-memory sliding-window rate limiter (`asyncio.Lock` + `defaultdict`); `is_rate_limited(key, max, window)` |
| `models/note.py` | Unified Note model (notes + tasks) |
| `models/user.py` | `id`, `username`, `email`, `password_hash` (nullable — NULL for OAuth-only), `oauth_sub` (unique nullable), `role`, `session_version` |
| `models/conversation.py` | `Conversation` + `Message` models |
| `models/app_log.py` | `AppLog` with `category`, denormalised `username`, `details` JSONB |
| `models/api_key.py` | `ApiKey`: `id`, `user_id`, `prefix`, `key_hash` (SHA-256), `name`, `scope` (`read`/`write`), `created_at`, `last_used_at` |
| `models/event.py` | Internal events store (`Event` model: `id`, `user_id`, `title`, `description`, `start_dt`, `end_dt`, `all_day`, `location`, `caldav_uid` nullable, `color`, timestamps) |
| `routes/api.py` | `/api` blueprint; `GET /api/health` (public) |
| `routes/auth.py` | Register, login, logout, me, password/email change, password reset, invite registration, OAuth login+callback; rate limiting; `LOCAL_AUTH_ENABLED` guards |
| `routes/admin.py` | Backup, restore, user management, registration toggle, invitations, base URL, SMTP (admin only) |
| `routes/chat.py` | Conversations CRUD; SSE generation stream; model pull/delete/list/warm |
| `routes/notes.py` | Notes CRUD + wikilinks + backlinks + assist + link suggestions + version history + drafts |
| `routes/tasks.py` | Tasks CRUD; `POST` accepts `project` name string (resolved to `project_id`) |
| `routes/task_logs.py` | Task work log CRUD (`GET/POST/DELETE /api/tasks/:id/logs`) |
| `routes/projects.py` | Projects CRUD + summary endpoint |
| `routes/milestones.py` | Milestones CRUD under `/api/projects/:id/milestones` |
| `routes/settings.py` | Per-user settings key-value (`GET/PUT /api/settings/:key`) |
| `routes/journal.py` | Journal config CRUD; today/day/days conversation accessors; weather endpoints (cached/current/refresh/geocode); moments CRUD; trigger-prep |
| `routes/groups.py` | Group CRUD + membership management (admin) |
| `routes/shares.py` | Share project/note with user or group; revoke; list incoming shares |
| `routes/in_app_notifications.py` | In-app notification list + mark-read + count |
| `routes/push.py` | Web Push subscription subscribe/unsubscribe; VAPID public key |
| `routes/users.py` | User profile; admin user list + delete |
| `routes/images.py` | Serve cached images at `/api/images/<id>` |
| `routes/export.py` | `GET /api/export` — personal Markdown ZIP or JSON array download |
| `routes/api_keys.py` | API key CRUD (`GET/POST/DELETE /api/api-keys`) |
| `mcp/server.py` | Mounts the in-app FastMCP server at `/mcp` (streamable HTTP, Bearer auth) |
| `routes/quick_capture.py` | `POST /api/quick-capture` — single-shot natural language item creation |
| `routes/search.py` | `GET /api/search` — semantic + keyword hybrid search |
| `services/auth.py` | `create_user`, `authenticate`, user lookups, password reset tokens, invitation tokens |
| `services/oauth.py` | OIDC discovery (cached), PKCE auth URL, code exchange, `find_or_create_oauth_user` |
| `services/api_keys.py` | `generate_key()`, `create_api_key()`, `list_api_keys()`, `revoke_api_key()`, `lookup_key()` (SHA-256 hash lookup) |
| `services/llm.py` | `build_context()`, RAG injection, history summarisation, `stream_chat_with_tools()`, URL fetching, SSRF guard |
| `services/generation_task.py` | `run_generation()` — full chat pipeline: intent routing, tool loop, SSE fan-out, push notification; `run_assist_generation()` |
| ~~`services/intent.py`~~ | Removed — intent routing eliminated; the main model handles all tool routing directly |
| `services/tools/` | LLM tool package — decorator-based registry (`_registry.py`), shared helpers (`_helpers.py`), and one module per domain: `notes.py` (create/update/delete/search/list/read), `tasks.py` (list/log_work), `entities.py` (save_person/save_place/lists), `projects.py`, `calendar.py`, `web.py`, `rag.py`, `profile.py`, `rss.py`, `weather.py`, `utility.py` (calculate). `execute_tool()` and `get_tools_for_user()` are the public API. |
| `services/projects.py` | Project CRUD + `generate_project_summary()` (Ollama, fire-and-forget) + `backfill_project_summaries()` (startup) |
| `services/embeddings.py` | `upsert_note_embedding()`, `semantic_search_notes(orphan_only=False)` (pgvector cosine similarity) |
| `services/generation_buffer.py` | In-memory SSE event buffer; `cancel_event`; 60s cleanup; supports both chat (int keys) and assist (string keys) |
| `services/notes.py` | Note CRUD, wikilink resolution, backlink queries, tag management |
| `services/note_versions.py` | Version snapshot on save; restore-from-version; diff metadata |
| `services/note_drafts.py` | Per-user per-note draft persistence (AI Assist pending state) |
| `services/settings.py` | `get_setting()`, `set_setting()`, `get_all_settings()` — key-value per user |
| `services/tag_suggestions.py` | `/api/notes/suggest-tags` — LLM-generated tag suggestions for note body |
| `services/access.py` | Permission resolution for all shared resources |
| `services/sharing.py` | Create/revoke shares; list shares; `get_shared_with_me()` |
| `services/groups.py` | Group CRUD; membership management |
| `services/notifications.py` | Create/read/mark-read in-app notifications |
| `services/task_logs.py` | Append/list/delete work log entries on tasks |
| `services/journal_prep.py` | Deterministic data gather (tasks/events/weather/projects/recent moments/open threads) → LLM prose opener; persisted as the first assistant message of today's journal Conversation |
| `services/journal_pipeline.py` | System-prompt builder for journal conversations; calls `build_profile_context()` so the LLM sees the user's profile + learned summary |
| `services/journal_scheduler.py` | APScheduler `BackgroundScheduler`; per-user prep job; live-reschedule via `update_user_schedule()`; catch-up logic for missed runs |
| `services/journal_search.py`, `services/moments.py` | Moment recording + search across journal history |
| `services/user_profile.py` | `build_profile_context()` consolidates profile + observations + learned_summary into a system-prompt block |
| `services/research.py` | SearXNG research pipeline: sub-queries → parallel fetch → outline → section synthesis → executive summary → index note with linked section notes |
| `services/events.py` | Internal events CRUD: `list_events`, `create_event`, `update_event`, `delete_event`, `get_event`; source of truth for all event LLM tools |
| `routes/events.py` | `/api/events` — event CRUD routes |
| `services/caldav.py` | Optional CalDAV sync — user-configured external server; syncs to/from internal store via `caldav_uid` FK; `is_caldav_configured()` guards tool activation |
| `services/calendar_sync.py` | Dead code — Radicale sync service; was trialled and removed |
| `services/images.py` | `fetch_and_store_image()` — SHA-256 dedup, content-type validation, 5 MB cap |
| `services/backup.py` | `export_full_backup()`, `export_user_backup()`, `restore_full_backup()` (version 2 with ID maps) |
| `services/push.py` | VAPID key auto-generation; `send_push_notification()` fire-and-forget; 410 Gone cleanup |
| `services/logging.py` | `log_audit`, `log_usage`, `log_error`, `start_log_retention_loop` (hourly cleanup) |
| `services/email.py` | SMTP email sending for password reset; reads `SMTP_*` env vars |
| `services/weather.py` | Nominatim geocoding + Open-Meteo forecast + per-user DB cache |
| `services/rss.py` | feedparser fetch; per-feed DB cache; prune-to-100 items |
| `services/assist.py` | `build_assist_messages()` — section/whole-doc modes; project context injection |
### Frontend (`frontend/src/`)
| File | Responsibility |
|------|---------------|
| `App.vue` | App shell (`100dvh` flex column); global keyboard shortcuts (`g`+key nav, `n/t/c/e///?`); starts status polling + loads settings on mount |
| `api/client.ts` | `ApiError`, `apiGet/Post/Put/Patch/Delete`, `apiSSEStream` (fetch + ReadableStream), auto 401→login redirect |
| `stores/auth.ts` | User, `isAuthenticated`, `isAdmin`, `oauthEnabled`, `localAuthEnabled`, login/register/logout/checkAuth |
| `stores/chat.ts` | Conversation CRUD; `sendMessage()` SSE streaming; `reconnectIfGenerating()`; message queue (localStorage); `streamingStatus` |
| `stores/notes.ts` | CRUD + tag filter; `resolveTitle`; `convertToTask/Note`; `fetchBacklinks`; `fetchAllTags` |
| `stores/tasks.ts` | CRUD + status/priority filter; `patchStatus` |
| `stores/settings.ts` | `assistantName`, `defaultModel`, `installedModels`; `pullModel()`, `deleteModel()` |
| `stores/push.ts` | `isSupported`, `permission`, `isSubscribed`, `subscribe/unsubscribe` |
| `stores/notifications.ts` | `count`, `items`, `fetchCount`, `fetchAll`, `markRead`, `markAll` |
| `views/HomeView.vue` | Chat-first dashboard; 6 task sections (Overdue → Other); 8 recent notes; inline streaming response |
| `views/ChatView.vue` | `/chat` — SSE streaming; context sidebar (Suggested/In Context); note picker; scope chip (RAG project scope pill above input); message queue; bulk delete |
| `views/CalendarView.vue` | `/calendar` — FullCalendar v6 month/week/day views; click to create event; click event to open EventSlideOver |
| `components/EventSlideOver.vue` | Reusable slide-over for event create/edit/delete; used in ToolCallCard, HomeView, CalendarView |
| `views/WorkspaceView.vue` | `/workspace/:id` — 3-panel (tasks/chat/notes); SSE tool-call watcher; conv persisted to localStorage |
| `views/GraphView.vue` | `/graph` — D3 force-directed; tag/note/project-hub nodes; physics panel; peek panel |
| `views/ProjectView.vue` | Kanban grouped by milestone; advance buttons; milestone management |
| `views/SettingsView.vue` | Tabbed settings (11 tabs); tab state in localStorage |
| `views/NoteEditorView.vue` | Tiptap editor; 2-column layout; AI assist panel; diff view; version history; autosave |
| `views/TaskEditorView.vue` | Tiptap editor; 2-column layout; AI assist panel; task log section; sub-tasks |
| `components/ToolCallCard.vue` | Tool call results; `requires_confirmation` inline confirm/deny; direct POST on "Create anyway" |
| `components/ChatMessage.vue` | Message bubble; markdown rendering; tool call cards; "Save as Note" |
| `components/TagInput.vue` | Chip-based tag input; Enter/comma to confirm; autocomplete from `/api/notes/tags` |
| `components/TiptapEditor.vue` | Tiptap wrapper; markdown↔HTML round-trip; selection change emit; WikilinkDecoration; TagDecoration |
| `components/ShareDialog.vue` | `<Teleport>` modal; user tab + group tab; current shares list |
| `components/WorkspaceTaskPanel.vue` | Milestone-grouped task list; detail slide-over |
| `components/WorkspaceNoteEditor.vue` | List ↔ TipTap editor with autosave |
| `composables/useAssist.ts` | AI assist: section parsing, SSE streaming, accept/reject, LCS diff, persistent draft |
| `composables/useAutoSave.ts` | Interval-based autosave (5 min) with dirty/saving guards |
| `composables/useEditorGuards.ts` | Ctrl+S, `beforeunload` warning, route-leave confirm |
| `composables/useTagSuggestions.ts` | `/api/notes/suggest-tags` call + suggestion state |
| `composables/useListKeyboardNavigation.ts` | j/k/Enter list navigation; focus-in-input guard |
| `extensions/WikilinkDecoration.ts` | ProseMirror decoration plugin highlighting `[[wikilinks]]` |
| `extensions/TagDecoration.ts` | ProseMirror decoration plugin highlighting `#tags` |
| `extensions/WikilinkSuggestion.ts` | `@tiptap/suggestion` extension for `[[` autocomplete |
## Key Services
| Service | Responsibility |
|---------|---------------|
| `services/access.py` | Permission resolution for all shared resources |
| `services/llm.py` | `build_context()`, RAG injection, history summarisation |
| `services/generation_task.py` | SSE streaming, tool-call loop, GenerationBuffer management |
| `services/tools/` | LLM tool implementations (38 tools across 11 modules); decorator-based registry |
| `services/embeddings.py` | `upsert_note_embedding()`, `semantic_search_notes()` |
| `services/journal_prep.py` | Data gather → LLM prose opener; persisted into today's journal conversation |
| `services/journal_scheduler.py` | APScheduler integration, per-user prep job, catch-up for missed runs |
| `services/backup.py` | Full and per-user backup export/restore (version 2 format) |
| `services/weather.py` | Nominatim geocoding + Open-Meteo forecast fetch + DB cache |
| `services/rss.py` | feedparser-based fetch, per-feed DB cache, prune-to-100 |
## Authentication
- **Sessions**: `HttpOnly`, `SameSite=Lax` cookies, `Secure` when `SECURE_COOKIES=true`. Changing a password bumps the user's session version and signs out their other sessions.
- **API keys**: `Authorization: Bearer fmcp_...`, stored hashed. A `read` key may only call read endpoints and read MCP tools.
- **OIDC** single sign-on with PKCE, optionally replacing local logins ([SSO / OAuth](sso-oauth.md)).
- **Rate limits** on register, login and forgot-password, per client IP (`TRUST_PROXY_HEADERS` reads the IP from the proxy's headers).
**Session cookies** — `HttpOnly`, `SameSite=Lax`, optionally `Secure` (`SECURE_COOKIES` env var). Session includes `session_version`; mismatch with DB value (after password change) results in 401 and session clear.
Credentials Grimoire stores for outside services (forge tokens, the backup destination's S3 secret) are sealed under a key derived from `SECRET_KEY`, which the instance makes on first start and keeps in `/data/secret_key` unless one is set.
**Bearer token / API keys** — `Authorization: Bearer <key>` accepted by `_check_auth()` as an alternative to session cookies. The raw key is SHA-256 hashed and looked up via `services/api_keys.py`. Keys with `scope=read` are rejected on non-safe methods (`POST`/`PATCH`/`DELETE`). Used by Fable MCP and any external API consumers.
## Repository layout
**Local auth + OIDC/OAuth (PKCE)** — `services/oauth.py` handles discovery and `find_or_create_oauth_user`. On OAuth login: checks existing `oauth_sub` → matching email → creates new user.
```
├── alembic/versions/ database migrations
├── src/grimoire/
│ ├── app.py app factory, blueprints, startup and shutdown
│ ├── config.py environment configuration
│ ├── models/ SQLAlchemy models
│ ├── routes/ REST blueprints
│ ├── mcp/ MCP server and its tool modules
│ ├── services/ shared logic
│ └── restore.py `python -m grimoire.restore` for a fresh instance
├── frontend/src/ views, components, stores, composables, api client
├── plugin/ Claude Code hooks, skills and commands
├── scripts/ CI checks and the plugin version minter
└── tests/ unit tests, plus `-m integration` against real Postgres
```
See [sso-oauth.md](sso-oauth.md) for provider-specific setup instructions.
## LLM Pipeline Internals
### Tool Routing
No separate intent router — the main model handles all tool routing directly via Ollama's structured tool-calling output. The model receives the full tool schema list and decides whether to call a tool or respond conversationally. Extended reasoning (`think=True`) is always on for qwen3-class models: content-based gating was tried but exposed tool-call template fragility on short tool-intent prompts.
### Tool Loop
Multi-round tool loop (max 5 rounds). All implementations in `services/tools/` (decorator-based registry); `execute_tool(user_id, tool_name, arguments, conv_id=None, workspace_project_id=None)` is the dispatcher. `conv_id` and `workspace_project_id` are threaded in from `run_generation()` so tools like `set_rag_scope` can write to the current conversation.
**Unified `create_note` tool** — creates both notes and tasks. Setting `status` (e.g. `"todo"`) creates a task; omitting it creates a knowledge note. All task fields (due_date, priority, milestone, parent_task, recurrence_rule) are available on the single tool.
**Duplicate protection on `create_note`:**
1. Exact title match (case-insensitive) → hard block, redirect to `update_note`
2. Fuzzy title match (SequenceMatcher ≥ 82%; punctuation stripped before candidate search) → hard block
3. Semantic content similarity (threshold 0.90, body ≥ 80 chars) → soft block with `requires_confirmation: true`
**Project resolution** (`_helpers.resolve_project`): 4-step lookup — (1) exact DB match, (2) `query in title` substring, (3) `title in query` reverse substring, (4) SequenceMatcher ≥ 0.55.
### Context Window and Summarisation
`OLLAMA_NUM_CTX` (default 16384) controls the context window for all generation calls. Intent classification always uses `num_ctx=4096` to reduce VRAM pressure.
History summarisation threshold: 30 messages. Keeps 8 recent messages. Summary max 400 tokens.
### Web Research Pipeline
`services/research.py` implements a full autonomous research pipeline:
1. Intent model generates 5 focused sub-queries
2. All 5 SearXNG queries run in parallel (200ms stagger to avoid rate limiter)
3. Up to 15 unique URLs fetched in parallel
4. LLM generates an outline (2–8 sections with title + focus)
5. Each section synthesised in parallel from relevant sources
6. Executive summary generated from all section content
7. Index note created with executive summary + links to section notes; section notes linked back via `parent_id`
8. Falls back to single-note synthesis if outline generation fails
SearXNG tip: add the app server IP to `botdetection.ip_lists.pass_ip` in SearXNG `settings.yml` to bypass the rate limiter for trusted backend requests.
### Image Cache
`search_images` tool fetches images server-side via SearXNG, stores them on disk (SHA-256 dedup, content-type validation, 5 MB cap), and serves from `/api/images/<id>`. The user's browser never contacts the original image host.
Config: `IMAGE_CACHE_DIR` (default `/data/images`), `IMAGE_MAX_BYTES` (default 5 MB).
## RAG Pipeline
1. `semantic_search_notes()` — cosine similarity via pgvector, threshold configurable per call; accepts `orphan_only` and `project_id` scope flags.
2. Notes ≥ 0.60 similarity auto-injected into system prompt (up to 3, 800 chars each).
3. Notes 0.45–0.60 surfaced in chat sidebar as "Suggested" (user clicks to include).
4. Explicitly included notes delivered full-body.
5. `excluded_note_ids` prevents the current note from being injected as its own context.
### RAG Scope (three-value system)
`conversations.rag_project_id` controls which notes are eligible for retrieval:
| Value | Behaviour |
|-------|-----------|
| `NULL` (default) | Orphan notes only — notes with `project_id IS NULL` |
| `-1` | All notes — opt-in to global search |
| Positive int | That project's notes only |
`build_context()` derives `orphan_only` + `effective_project_id` from this value before calling both the semantic and keyword search paths.
**Scope tools** — Two LLM tools let the model discover and switch scope mid-conversation:
- `search_projects` — SequenceMatcher scoring over title + description + `auto_summary`; returns top 5 matching projects.
- `set_rag_scope` — persists the new `rag_project_id` to DB immediately; blocked in workspace view; causes the SSE `done` event to include `new_rag_scope` + `new_rag_scope_label` so the frontend chip updates reactively.
**Project summaries** — `generate_project_summary()` calls Ollama (fire-and-forget) and stores the result in `projects.auto_summary`. Triggered on project update and note saves (debounced 1h). `backfill_project_summaries()` runs at startup.
Embedding model: `nomic-embed-text` via Ollama. Backfill runs 30s after startup (background task).
-164
View File
@@ -1,164 +0,0 @@
# Backups
Grimoire can back itself up every day to a **folder** or an **S3-compatible
bucket**. Each run adds one file; Grimoire never deletes an old one. Keeping and
thinning the history is the destination's job.
Set it up under **Settings → Data → Scheduled backups** (admins only).
## What a backup is
One file per run, named for the moment it was taken in UTC:
```
grimoire-20261015T020000Z.tar
```
Backups written while the app was called Scribe are named `scribe-…tar`.
They are counted with the rest and restore the same way.
It is a plain, **uncompressed** tar:
- `manifest.json` comes first. It holds the backup format version, the Grimoire
version, when the backup was taken, and how many rows each table has.
- Then there is one `<table>.jsonl` per table, one row per line.
It holds everything the admin **Full Backup** download holds, laid out
differently. It does **not** hold:
- **Search indexes.** They are rebuilt from the records when Grimoire starts.
- **Sharing:** groups, project shares and note shares.
- **Credentials:** API keys and forge connections. After a restore, issue new
API keys and add forge connections again. (Forge tokens are also sealed under
`SECRET_KEY` in the database itself, so a copy of the database doesn't carry
them in readable form either. The key lives in `/data/secret_key`, outside
the database.)
- **Logs and notifications.**
- **The backup destination's own S3 secret.**
The file contains hashed passwords, so store it somewhere only trusted people
can read.
### Why it is shaped this way
The file is meant to be copied off-site by a deduplicating tool such as
restic, which stores a chunk of data only once no matter how many backups
contain it. Every choice serves that:
- **It is never compressed.** Compression changes every byte after the first
edit, so nothing would match the day before. restic compresses its own
repository after deduplicating, so nothing is lost.
- **Rows are in primary-key order, with their keys sorted.** A row that did
not change is the same bytes in the same place every day. An edited row
changes one line, and new rows land at the end of their table's file.
- **Each table is its own file**, so a busy table never shifts the bytes of a
quiet one.
- **The parts that change every run are kept in `manifest.json`**: the time
and the counts.
- **Each run gets a new, dated name.** Overwriting one name would keep one
copy and no history.
Day to day, the off-site copy then stores roughly what changed, not the whole
file again.
## Destinations
### A folder
A path **inside the Grimoire container**, normally a share mounted there:
```yaml
services:
grimoire:
volumes:
- /nfs/backup/grimoire:/backups
```
Grimoire writes each file under a hidden name (`.grimoire-….tar.part`) and renames
it once it is complete. A program watching the folder never sees half a
backup.
### An S3-compatible bucket
You need four things:
- **Address:** the service itself, like `https://backup.example.com`. Don't
include the bucket; Grimoire addresses buckets by path.
- **Bucket.**
- **Access key.**
- **Secret key.**
Grimoire uploads and lists; it never deletes. A key that may **only add
objects** is therefore enough, and it is the safer key to hand out: a stolen
one cannot wipe the history.
The secret is stored encrypted, under a key derived from `SECRET_KEY`. It is
never shown again, and never included in a backup. If an instance's
`SECRET_KEY` changes, for example after a restore onto a new instance, the
page asks for the secret again.
**Test destination** checks the destination without sending a backup:
- a folder: it writes and removes a hidden file;
- a bucket: it only lists, because an add-only key could never remove a test
file.
## Schedule and history
- Backups run once a day at the hour you choose, in UTC. The default is 02:00,
before the nightly trash purge and database maintenance.
- **Back up now** runs one straight away.
- The page leads with when the last good backup was taken. It says
**Overdue** when backups are on and none has worked for 26 hours.
- It lists the last 30 runs, and gives the reason for each failure in words.
- A run that hasn't finished after an hour is abandoned and recorded as
failed.
## Example: Fabled Archives
[Fabled Archives](https://git.fabledsword.com/bvandeusen/fabledarchives)
keeps service backups as "backup targets". Either destination works with it.
1. On Archives' **Backups** page, make a target for Grimoire, named `grimoire`
for example.
2. Choose how Grimoire's files arrive:
- **S3**: turn on the S3 way in for the target and make an **append-only**
S3 key. In Grimoire, choose **An S3 bucket**:
- address: Archives' backup address (`http://<archives>:8081`);
- bucket: the target's name;
- access key and secret key: the key you just made.
- **A folder**: mount the share Archives watches into Grimoire as
`/backups/grimoire`, and point the target at that folder. In Grimoire, choose
**A folder** with `/backups/grimoire`.
3. Set the target's **Retention** to **Archives**. Archives then keeps the
newest backups and thins the rest by day, week and month. Grimoire only adds.
4. Give the target an expected cadence of **daily**, so a missed backup turns
it late in Archives as well as overdue in Grimoire.
5. Add an off-site **restic** replication job for the target. The uncompressed
layout above is what lets that job send only what changed each day.
## Restoring
### Into a fresh instance
Start Grimoire on an empty database, copy the backup file into the container,
and run:
```sh
docker exec -it grimoire python -m grimoire.restore /backups/grimoire-20261015T020000Z.tar
```
- The command refuses a database that already has an account. A restore
re-creates everything with new ids, so on a live instance it would add a
second copy instead of replacing anything.
- Nothing limits the file's size.
- It also reads the JSON from the admin **Full Backup** download.
Afterwards, **restart Grimoire** so the search indexes are rebuilt. Then issue
new API keys, add forge connections again, and set up any sharing.
### From the web
**Settings → Data → Restore from Backup** takes the `.tar` as well as the
JSON download. The upload is held to the server's request-size limit. For a
bigger file, use the command above.
+99 -73
View File
@@ -1,103 +1,108 @@
# Configuration
Grimoire reads its configuration from environment variables. Most settings people change day to day (registration, SMTP, retrieval thresholds, backups, forge connections) live in **Settings** in the web UI instead.
Configuration is via environment variables. The `docker-compose.yml` file sets defaults for local development; override them in a `.env` file (gitignored).
Any variable marked *secret* can also be read from a file: set `<NAME>_FILE` to the path, for example a Docker secret at `/run/secrets/grimoire_key`.
## Environment variables
## Environment Variables
### Core
| Variable | Default | Description |
|----------|---------|-------------|
| `DATABASE_URL` *(secret)* | `postgresql+asyncpg://grimoire:grimoire@localhost:5432/grimoire` | PostgreSQL connection string, using the `asyncpg` driver |
| `SECRET_KEY` *(secret)* | kept in `DATA_DIR` | Signs sessions and seals stored credentials. Unset, the instance makes its own on first start and keeps it in `DATA_DIR/secret_key`. Set it only to manage the key yourself; it wins over the kept one (see below) |
| `DATA_DIR` | `/data` | Where the instance keeps its own `SECRET_KEY`. Mount a volume here |
| `BASE_URL` | `http://localhost:5000` | Public URL, used in email links and OIDC redirects |
| `SECURE_COOKIES` | `false` | `true` when Grimoire is served over HTTPS |
| `TRUST_PROXY_HEADERS` | `false` | `true` behind a reverse proxy, to read the client IP from `X-Forwarded-For` / `X-Real-IP` |
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING` or `ERROR` |
| `LOG_RETENTION_DAYS` | `90` | Days of app logs to keep |
| `FASTEMBED_CACHE_DIR` | `/data/fastembed-cache` | Where the embedding model is stored after its first download |
| `DATABASE_URL` | `postgresql+asyncpg://fabled:fabled@db/scribe` | PostgreSQL async connection string |
| `SECRET_KEY` | `dev-secret-change-me` | Session signing key — **change this in production** |
| `SECRET_KEY_FILE` | — | Path to a Docker secret file containing the key (alternative to `SECRET_KEY`) |
| `LOG_LEVEL` | `INFO` | Logging verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
| `SECURE_COOKIES` | `false` | Set `true` when running behind TLS |
| `BASE_URL` | — | Public URL (e.g. `https://notes.example.com`) — required for OIDC redirect URIs and email links |
### Sign-in
### LLM / Ollama
| Variable | Default | Description |
|----------|---------|-------------|
| `LOCAL_AUTH_ENABLED` | `true` | `false` turns off username/password login, leaving SSO only |
| `OIDC_ISSUER` | — | OIDC issuer URL |
| `OLLAMA_URL` | `http://ollama:11434` | Ollama API base URL |
| `OLLAMA_MODEL` | `llama3.2` | Default LLM model (used as fallback; per-user setting overrides this) |
| `EMBEDDING_MODEL` | `nomic-embed-text` | Model used for semantic search / RAG embeddings |
| `OLLAMA_NUM_CTX` | `16384` | Context window size passed to Ollama for all generation calls |
### Authentication / OIDC
| Variable | Default | Description |
|----------|---------|-------------|
| `LOCAL_AUTH_ENABLED` | `true` | Set `false` to disable local username/password login (SSO-only mode) |
| `OIDC_ISSUER` | — | OIDC issuer URL (e.g. `https://auth.example.com/application/o/fabled/`) |
| `OIDC_CLIENT_ID` | — | OIDC client ID |
| `OIDC_CLIENT_SECRET` *(secret)* | — | OIDC client secret |
| `OIDC_SCOPES` | `openid profile email` | Scopes to request |
| `OIDC_CLIENT_SECRET` | — | OIDC client secret |
| `OIDC_CLIENT_SECRET_FILE` | — | Docker secret file alternative to `OIDC_CLIENT_SECRET` |
| `OIDC_SCOPES` | `openid profile email` | Space-separated OIDC scopes to request |
SSO turns on when the issuer, client ID and secret are all set. See [SSO / OAuth](sso-oauth.md).
See [sso-oauth.md](sso-oauth.md) for provider-specific setup.
### Email
Admins can set SMTP in **Settings**; these are the defaults until they do.
| Variable | Default |
|----------|---------|
| `SMTP_HOST` | — |
| `SMTP_PORT` | `587` |
| `SMTP_USERNAME` | — |
| `SMTP_PASSWORD` *(secret)* | — |
| `SMTP_FROM_ADDRESS` | — |
| `SMTP_FROM_NAME` | `Fabled Grimoire` |
| `SMTP_USE_TLS` | `true` |
### Integrations
### Security / Proxy
| Variable | Default | Description |
|----------|---------|-------------|
| `PLUGIN_MARKETPLACE_URL` | this repo's git URL | The git URL shown in the plugin install command under Settings → MCP Access. An admin can override it in Settings |
| `FORGE_KIND`, `FORGE_BASE_URL`, `FORGE_TOKEN` *(secret)* | — | A git forge connection for admins' projects, so a token can live in a Docker secret. Each user can add their own under Settings → Integrations; a stored connection for the same host wins |
| `FORGE_WEBHOOK_SECRET` *(secret)* | — | Verifies push webhooks from the forge |
| `SEARXNG_URL` | — | A SearXNG instance, listed as the web-search backend under Settings → Integrations |
| `TRUST_PROXY_HEADERS` | `false` | Set `true` when behind a trusted reverse proxy to read real IP from `X-Forwarded-For` / `X-Real-IP` |
## Running it
### Web Search / Images
### Docker Compose
| Variable | Default | Description |
|----------|---------|-------------|
| `SEARXNG_URL` | — | SearXNG base URL for web search and image search tools |
| `IMAGE_CACHE_DIR` | `/data/images` | Directory for cached search images |
| `IMAGE_MAX_BYTES` | `5242880` (5 MB) | Maximum size for cached images |
[`docker-compose.yml`](../docker-compose.yml) runs the published image with its own database, and needs nothing set:
### Data / Logging
| Variable | Default | Description |
|----------|---------|-------------|
| `LOG_RETENTION_DAYS` | `90` | Days to keep app logs before automatic pruning |
| `DATA_DIR` | `/data` | Root directory for persistent data (VAPID keys, backups) |
## Docker Compose Setup
### Development (`docker-compose.yml`)
```bash
docker compose up -d
# Copy the example env file
cp .env.example .env
# Edit .env to set a real SECRET_KEY
# Start the stack
docker compose up --build
# App available at http://localhost:5000
```
Open `http://localhost:5000`. The first account registered becomes the admin; after that, registration stays closed until an admin opens it in Settings or sends an invite.
The first user to register becomes admin. Registration auto-closes after that (re-enable from Settings → Users).
Update with `docker compose pull && docker compose up -d`. The key and the data stay in their volumes.
### Production (`docker-compose.prod.yml`)
### From source
The production compose file adds:
- Docker Secrets for `SECRET_KEY_FILE` and `DATABASE_URL_FILE`
- Network isolation (internal bridge for DB + Ollama; only the app is externally accessible)
- Health checks and resource limits
```bash
docker compose -f docker-compose.dev.yml up --build
# Create Docker secrets
echo "$(python3 -c 'import secrets; print(secrets.token_hex(32))')" | docker secret create fabled_secret_key -
echo "postgresql+asyncpg://fabled:strongpassword@db/scribe" | docker secret create fabled_db_url -
# Deploy
docker stack deploy -c docker-compose.prod.yml fabled
```
### Docker Swarm
## Production Deployment
[`docker-compose.prod.yml`](../docker-compose.prod.yml) is a stack file: the app and database on a private overlay network, with `SECURE_COOKIES` and `TRUST_PROXY_HEADERS` on. A Swarm can move the app to another node, away from a node-local `/data` volume, so the stack sets its own key: provide `DB_PASSWORD` and `SECRET_KEY` in the environment (or mount a Docker secret and set `SECRET_KEY_FILE`):
### Reverse Proxy (Required)
```bash
DB_PASSWORD=... SECRET_KEY=... docker stack deploy -c docker-compose.prod.yml grimoire
```
Fabled Assistant does **not** handle SSL/TLS. Run it behind a reverse proxy:
The database user, database and volume default to `grimoire`. An install created under the name Scribe keeps its data by setting `POSTGRES_USER`, `POSTGRES_DB` and `PGDATA_VOLUME` to the names it was created with.
The shipped files leave out healthchecks and restart policies on purpose. The app waits for its database on startup, and the orchestrator's defaults restart it if it ever exits.
### Volumes
- **Database** (`pgdata`): everything Grimoire stores.
- **App data** (`/data`): the instance's own `SECRET_KEY`, the embedding model cache and diagnostics. Losing it costs the key: Settings then asks for saved forge tokens and the backup secret again, and the model downloads again.
Back up the database with **Settings → Data → Scheduled backups** ([Backups](backups.md)). A database backup doesn't carry the key; to move sealed credentials to a new instance, copy `/data/secret_key` with it.
## Behind a reverse proxy
Grimoire doesn't terminate TLS. Put it behind Nginx, Traefik or Caddy, forward to port 5000, and keep port 5000 off the internet. Set `SECURE_COOKIES=true`, `TRUST_PROXY_HEADERS=true` and `BASE_URL`.
- **Nginx**, **Traefik**, or **Caddy** in front of the app container
- Terminate TLS at the proxy; forward to port 5000
- **Do not expose port 5000 directly to the internet**
- Rate-limit auth endpoints: ≤ 5 req/min per IP on `/api/auth/login` and `/api/auth/register`
Example Nginx location block:
```nginx
location / {
proxy_pass http://127.0.0.1:5000;
@@ -106,18 +111,39 @@ location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# MCP responses can stream.
# Required for SSE streaming
proxy_buffering off;
proxy_read_timeout 600s;
proxy_send_timeout 600s;
}
```
With `TRUST_PROXY_HEADERS=true`, make sure the proxy replaces any `X-Forwarded-For` the client sends.
If using `TRUST_PROXY_HEADERS=true`, ensure your proxy strips any client-supplied `X-Forwarded-For` headers before adding its own.
## Security checklist
### Security Checklist
- **Keep the key.** Forge tokens and the backup destination's S3 secret are sealed under a key derived from `SECRET_KEY`, so a copy of the database alone can't read them. Unset, the instance makes a random key and keeps it in `/data/secret_key`, readable by its owner only; keep that volume. If the key changes (say, a restore onto an instance with a different key), Settings marks those credentials and asks for them again; nothing else breaks. The app warns at startup when `SECRET_KEY` is a public placeholder or `/data` is not a mounted volume, and refuses to start with `dev-secret-change-me` when `SECURE_COOKIES=true`.
- **`SECURE_COOKIES=true`** whenever Grimoire is served over HTTPS.
- **Registration** is open only for the first account. Invite people, or open it briefly from Settings.
- **Signing everyone else out**: changing a password ends that user's other sessions; Settings → Account has a button for it too.
- **API keys** are per user, shown once, and can be read-only. Revoke any you no longer use under Settings → MCP Access.
- **Strong `SECRET_KEY`** — generate with:
```bash
python3 -c "import secrets; print(secrets.token_hex(32))"
```
Or use Docker Secrets via `SECRET_KEY_FILE`.
- **`SECURE_COOKIES=true`** — must be set when running behind TLS.
- **`BASE_URL`** — set to your public URL; required for OIDC redirects and email links.
- **Registration** — auto-closes after the first user (admin). Re-enable from Settings → Users or send invite links.
- **Session invalidation** — changing or resetting a password bumps `session_version`, evicting all other active sessions. Button also available in Settings → Account.
- **Keep Ollama on an internal network** — both compose files keep Ollama off the host network. Never expose the Ollama port publicly.
- **Default SECRET_KEY warning** — the app logs a `WARNING` on startup if the default dev key is in use.
## Model Management
Models are managed through Settings → General → Model Management in the web UI. You can:
- Pull new models by name (streams progress via SSE)
- View installed models with size and loaded/unloaded state
- Delete unused models
The app auto-warms user-preferred models on startup (only models already installed — never auto-pulls). The embedding model (`nomic-embed-text`) is auto-pulled on startup if missing.
Recommended models for 2× 8 GB GPU:
- **`qwen3:8b`** — strong reasoning + tools, fits in 8 GB, supports thinking mode
- **`qwen2.5:7b`** — fast, tool-capable, smaller context
- **`llama3.1:8b`** — reliable baseline, widely tested with tools
+223 -18
View File
@@ -53,7 +53,7 @@ Parchment is intentionally not pure white — it's slightly warm to feel like ag
| Bronze | `#8B7355` | Secondary action buttons (Cancel-but-not-destructive, alternate paths) |
| Pewter | `#3F4651` | Tertiary/ghost buttons |
**Critical rule:** Action button colors are universal across all apps. A Save button in Grimoire and a Save button in Minstrel look identical. Per-app accents do not appear on buttons.
**Critical rule:** Action button colors are universal across all apps. A Save button in Scribe and a Save button in Minstrel look identical. Per-app accents do not appear on buttons.
### Semantic colors
@@ -71,7 +71,7 @@ Parchment is intentionally not pure white — it's slightly warm to feel like ag
| App | Hex | Mood |
|---|---|---|
| Fabled Grimoire | `#5B4A8A` | Dusty violet — ink, manuscripts |
| Fabled Scribe | `#5B4A8A` | Dusty violet — ink, manuscripts |
| Minstrel | `#4A6B5C` | Forest teal — music, performance |
| Fabled Forge | `#8B5A2B` | Forge bronze — creation, craft |
| Roundtable | `#4A5D7E` | Slate blue — stewardship, infrastructure |
@@ -80,7 +80,7 @@ Parchment is intentionally not pure white — it's slightly warm to feel like ag
**Accent usage rules:**
- The accent appears on the app's wordmark and icon.
- The accent indicates active/current state in nav (the selected page, the active tab).
- The accent is the cursor color and text-selection color in long-form surfaces (Grimoire notes, Forge story drafts).
- The accent is the cursor color and text-selection color in long-form surfaces (Scribe notes, Forge story drafts).
- The accent does NOT appear on primary or secondary action buttons.
- The accent does NOT appear in body text or chrome.
- One accent per app. Don't mix accents within a single app.
@@ -101,8 +101,6 @@ All text-on-surface combinations meet WCAG AA at minimum. Parchment on Obsidian
| Inter | Body, UI, labels | Google Fonts |
| JetBrains Mono | Code, terminal output, monospaced data | Google Fonts |
All three are served by the app itself, from `frontend/src/assets/fonts/`, with every subset Google Fonts publishes for them. A page never asks a font host for anything, so an instance with no internet still draws in its own type. They are under the SIL Open Font License; the licences sit beside the files.
### Why this pairing
Fraunces is a contemporary serif with personality — it has the warmth and authority of a book serif without feeling like costume. It signals "this is considered" without signaling "this is a fantasy product." Inter is the workhorse — neutral, ubiquitous, designed for screens, doesn't compete with the serif. JetBrains Mono is the natural choice for any developer-adjacent product and supports ligatures.
@@ -126,7 +124,7 @@ Fraunces is a contemporary serif with personality — it has the warmth and auth
- **Sentence case everywhere.** Never Title Case for headings, never ALL CAPS except for the Tiny micro-label style.
- **Two weights only:** 400 regular and 500 medium. Never 600 or 700 — they read heavy in dark mode.
- **Fraunces only at 18px and above.** Below that it loses too much detail and feels fragile. For h3 and below, use Inter.
- **Line height** 1.5 for body, 1.3 for headings, 1.7 for long-form reading surfaces (Grimoire notes, Forge drafts).
- **Line height** 1.5 for body, 1.3 for headings, 1.7 for long-form reading surfaces (Scribe notes, Forge drafts).
- **Letter-spacing** at default for everything except the Tiny micro-label, which gets `0.08em` letter-spacing and uppercase styling.
---
@@ -175,7 +173,7 @@ Padding: `8px 16px` for default, `6px 12px` for compact, `10px 20px` for promine
Used for tags, hashtags, code spans, status badges. Background is the accent color at ~15% opacity, text is the accent at full strength. Border-radius 4px, padding `2px 8px`, Inter 11px/500.
In Grimoire specifically, hashtags and tags use the dusty violet accent. In Minstrel, they'd use forest teal. The pattern is shared; the color follows the app.
In Scribe specifically, hashtags and tags use the dusty violet accent. In Minstrel, they'd use forest teal. The pattern is shared; the color follows the app.
### Cards
@@ -274,9 +272,9 @@ Use **Lucide** (https://lucide.dev) as the base icon set — it matches this sty
## Per-app application
### Fabled Grimoire (#5B4A8A — dusty violet)
### Fabled Scribe (#5B4A8A — dusty violet)
A second-brain notes and task management tool. The accent appears in: the wordmark, hashtags and tag pills, the active nav item, text selection color, and the cursor in the editor. Notes are presented on Iron-surfaced cards with generous reading line-height. The hashtag system uses Grimoire's accent for visual continuity.
A second-brain notes and task management tool. The accent appears in: the wordmark, hashtags and tag pills, the active nav item, text selection color, and the cursor in the editor. Notes are presented on Iron-surfaced cards with generous reading line-height. The hashtag system uses Scribe's accent for visual continuity.
### Minstrel (#4A6B5C — forest teal)
@@ -334,7 +332,7 @@ Home server management. The accent appears on: the wordmark, healthy/online stat
}
/* Per-app accent — set ONE of these on the root for each app */
[data-app="grimoire"] { --fs-accent: #5B4A8A; }
[data-app="scribe"] { --fs-accent: #5B4A8A; }
[data-app="minstrel"] { --fs-accent: #4A6B5C; }
[data-app="forge"] { --fs-accent: #8B5A2B; }
[data-app="roundtable"]{ --fs-accent: #4A5D7E; }
@@ -356,13 +354,220 @@ If using Tailwind, extend the theme with these tokens rather than relying on def
---
# Grimoire
*Last updated: April 25, 2026. Iterate as the family of apps grows.*
The design system Grimoire renders with is stored in Grimoire itself: **Grimoire**, which inherits **FabledSword**, edited at `/design-systems`. `frontend/src/assets/theme.css` is generated from it, and CI rejects a `var()` the sheet doesn't define. Where this page and the stored system differ, the stored system is right. Grimoire applies the baseline above with these choices:
---
- **Accent footprint.** Dusty violet marks Grimoire's own moments: the wordmark, active navigation, the editor's cursor and selection, tags and wikilinks, in-progress status, focus rings, and the call to action that starts something new (a first note, say). Operating the software uses the action colours: Moss to save or confirm, Bronze for alternatives, Oxblood with an icon to delete, Pewter ghost for tertiary actions.
- **Light and dark.** Both are supported. Light mode is warm parchment (`#F5F1E8` page, `#FBF8F0` cards, `#14171A` text) rather than white, with a cool wash on code blocks so syntax colours stay readable. The accent, action and semantic colours are the same in both.
- **Status and priority.** Status: todo Pewter, in progress the accent, done Moss, cancelled Ash, paused Warning. Priority runs cool to warm: low Info, medium Warning, high Error. Overdue uses Error. Each has a low-alpha `-bg` companion.
- **Type.** Inter for UI and body, Fraunces at 18px and up (display, H1, H2), JetBrains Mono for code. Two weights only, 400 and 500. Line height 1.5 for body, 1.3 for headings, 1.7 on reading surfaces such as a note.
- **Icons.** Lucide, at 16px inline and 24px for navigation and primary actions; no emoji in the interface.
- **Borders are structural.** Cards, dialogs, code blocks, focus rings and major dividers get borders; list rows, chips and empty states are separated by surface tint and spacing instead.
# Scribe-specific decisions in progress
> This section tracks decisions made while adapting the FabledSword baseline above for Scribe specifically. Items here are *in progress* — once they feel solid, they get folded into the main body of the document (either as Scribe-specific extensions in the per-app section, or as updates to the universal rules where Scribe's needs reveal a gap in the baseline).
*Iteration started: 2026-04-26. Foundation pass shipped 2026-04-27 in `7a9a8b7` (palette, fonts, light mode, action tokens, hardcoded indigo cleanup, warm-gold deprecation). Surface phase shipped 2026-04-27 across `93a3beb` → `3c1ec40` (Lucide migration, Hybrid-rule button reclassification per surface, long-form line-height, two-weights-only). The system is now applied end-to-end; this section will fold into the main body once the result has had time to settle in real use.*
## Decisions made so far
### Accent footprint — Hybrid rule (not Strict)
The doc baseline says the per-app accent only appears on wordmark, active nav, cursor, and text selection — never on action buttons. Scribe currently uses indigo on essentially every interactive surface (CTAs, scrollbars, glows, borders, focus rings). Hard-cutting to the doc baseline would lose too much identity in one swing.
**Hybrid rule:** the accent reserves a slightly larger footprint than the doc baseline, but still much smaller than today.
- **Accent (dusty violet) lives on:** wordmark; active nav; cursor and text selection in editor surfaces; tags/pills/wikilinks; in-progress task badge; focus rings; **brand-moment CTAs** — chat Send, "Create note" empty-state CTA, journal Send, "Start journaling" empty-state.
- **Moss (sage-green primary) lives on:** Save / Submit / Confirm in forms and modals; generic affirmative actions where the button just means "do this thing" with no brand pretense.
- **Bronze (secondary):** Cancel-but-not-destructive, alternative paths.
- **Oxblood (destructive):** Delete / Remove (paired with an icon).
- **Pewter ghost:** tertiary actions, "later", "skip", "see also".
**Rule of thumb:** if the user is engaging with a *Scribe-feature moment* (sending a chat, opening a fresh note, jumping into the journal), accent. If they're just *operating the software* (saving an edit, confirming a dialog), Moss.
### Light mode — warm parchment, matched aesthetic
The doc is dark-only. Scribe today supports both light and dark, and we keep both. The light mode is derived to *match* the dark mode aesthetic rather than defaulting to system white-and-ink.
- Page background: in the `#F5F1E8` warm cream family (specific values TBD)
- Cards: near-white but slightly tinted
- Text: deep ink `#14171A` (mirroring Obsidian)
- Accent: same dusty violet `#5B4A8A` (works on both themes)
The metaphor stays consistent across themes: ink on aged paper (light) ↔ parchment text on graphite (dark). Light mode is *not* the system standard look.
**Known downside:** warm parchment backgrounds can fight with embedded color content. Mitigation: code blocks get a slight cool wash in light mode specifically, to keep syntax highlighting readable.
### Status and priority palette — extend the doc's semantic set
The doc's semantic colors (Success / Warning / Error / Info / Destructive) are leaner than what Scribe needs for task management. Rather than running a parallel palette, Scribe extends the doc by mapping its status/priority tokens onto doc primitives where they fit and defining new app-level tokens for the rest.
**Status (task lifecycle):**
| Token | Color | Source | Logic |
|---|---|---|---|
| `status-todo` | Pewter `#3F4651` | doc | Neutral, "not started yet" |
| `status-in-progress` | dusty violet `#5B4A8A` | accent | Active = brand moment per Hybrid |
| `status-done` | Moss `#4A5D3F` | doc Success | Affirmative completion |
| `status-cancelled` | Ash `#9C9A92` | doc | Faded, "let go" |
| `status-paused` | Warning `#8B6F1E` | doc | Stalled, needs attention — replaces the old warm gold treatment |
**Priority (loudness scale):**
| Token | Color | Source | Logic |
|---|---|---|---|
| `priority-low` | Info `#3D5A6E` | doc | Cool, FYI — quietest end of the spectrum |
| `priority-medium` | Warning `#8B6F1E` | doc | Golden, mid-attention |
| `priority-high` | Error `#C04A1F` | doc | Terracotta, urgent |
| `priority-none` | Vellum/Ash | doc | No signal |
The priority row reads as a clean cool→warm gradient (slate blue → golden brown → terracotta), which matches the semantic loudness — coherence the current ad-hoc palette doesn't have.
**Other functional tokens:**
| Token | Color | Logic |
|---|---|---|
| `wikilink` | dusty violet | Editorial brand moment per Hybrid |
| `overdue` | Error `#C04A1F` | Same as priority-high — overdue IS a priority signal |
| `toast-success` | Moss | doc semantic |
| `toast-error` | Error | doc semantic |
| `toast-info` | Info | doc semantic |
| `tag-bg` / `tag-text` | accent at 15% / accent | Per doc pill recipe |
Each token gets a `*-bg` companion at low alpha (matching the existing pattern in `theme.css`).
**Removed:** the warm gold accent (`--color-accent-warm: #b8860b`). Its two jobs split:
- Dates and timestamps (knowledge cards, event details, chat) → use `text-secondary` instead. Dates are metadata, not a brand surface; muted is the correct register.
- Paused project status → use the new `status-paused` (Warning `#8B6F1E`) row above. Same golden-brown family, semantically aligned.
### Typography — adopt the doc's stack and scale
Adopt the doc's type stack and scale verbatim, with one deferred verification (long-form line-height in practice).
- **Body font: Inter.** Replaces Scribe's current system-stack body font. Doc-defined; no Scribe-specific divergence.
- **Type scale:** as in the doc table — Display 40 / H1 32 / H2 24 / H3 18 / Body 15 / Body small 13 / Label 12 / Code 13 / Tiny 11.
- **Two weights only:** 400 regular, 500 medium. No 600/700 (reads heavy in dark mode and against the muted palette).
- **Family rules:** Fraunces at 18px+ only (Display, H1, H2). H3 and below = Inter. Code = JetBrains Mono.
- **Line height:** 1.5 body, 1.3 headings, **1.7 for long-form reading surfaces** (notes, journal entries).
- **Sentence case for everything**, except the Tiny micro-label style which gets uppercase + 0.08em letter-spacing.
Mechanical rollout — value swaps in `theme.css` plus loading Inter and JetBrains Mono from Google Fonts (Fraunces is already loaded).
### Light mode — concrete palette
Fills in the warm-parchment direction picked earlier. Treat these as starting values; tune in practice.
| Token | Hex | Role |
|---|---|---|
| Page bg | `#F5F1E8` | Warm cream — the "paper" |
| Card bg | `#FBF8F0` | Near-white, slightly warm — raised surfaces |
| Inset bg (inputs, code) | `#EFEAE0` | Slightly darker than page, "inset" feeling |
| Text primary | `#14171A` | Deep ink — Obsidian inverted |
| Text secondary | `#5A5852` | Warm mid-grey |
| Text muted | `#9A9890` | Warm light grey for hints/metadata |
| Border default | `#D9D6CE` | Warm light pewter, hairline weight |
**Code-block exception:** in light mode specifically, code blocks use a slight cool wash (e.g. `#EBEDF0`) instead of the warm inset bg, so syntax highlighting reads cleanly. This is the mitigation for the "warm bg fights colored content" downside.
The accent (`#5B4A8A` dusty violet), Moss, Bronze, Oxblood, and the semantic color set are **identical across themes** — only the surface and text palettes flip.
### Chat-bubble codification — keep the Illuminated Transcript pattern
The existing chat-bubble pattern (informally called "Illuminated Transcript") gets written into the design system as a documented chat component. Other apps in the family that add a chat surface inherit the pattern; Scribe's existing implementation continues to work with only color shifts.
**User bubble (whisper):**
- Background: transparent
- Border: 0.5px Pewter (was: indigo-tinted)
- Text color: secondary (Vellum dark / `#5A5852` light)
- Right-aligned, rounded except bottom-right (subtle "from-me" tail)
**Assistant bubble (lit):**
- Background: card surface (Iron dark / `#FBF8F0` light)
- Border: none on top/right/bottom; **2px solid accent (dusty violet) on left edge only**
- Box-shadow: accent-tinted glow + standard depth shadow (formula: `0 4px 28px rgba(<accent>, 0.14), 0 2px 8px rgba(0,0,0,0.4)` in dark; lower alphas in light)
- Text color: primary (Parchment dark / Obsidian-inverted light)
- Left-aligned, rounded except bottom-left
The 2px-accent left edge is the "illumination" — like an illuminated capital in a manuscript. The shadow is the lift. Together they make the assistant bubble read as the *primary* voice, while the user bubble is the *margin note*.
**Inline tool-call cards (`ToolCallCard`)** rendered inside an assistant bubble do NOT get their own border (per the border philosophy — the bubble already contains them). They use a slight surface tint to differentiate.
### Iconography — adopt Lucide, enforce a scale
Scribe currently hand-inlines SVG paths in 16+ Vue files, with 5 different stroke weights and 8+ different sizes. The visual style is already outline + rounded caps + `currentColor` stroke (matches the doc's intent), but there's no shared source and no scale discipline.
**Migration policy:**
1. **Install `lucide-vue-next`** as the icon source. Replace hand-inlined SVGs with imported components. Single source of truth.
2. **Strict size scale: 16px and 24px only.** Today's mix of 12/13/14/15/17/18/20 collapses to those two. 16 for inline-with-text and small affordances; 24 for nav and primary actions.
3. **Stroke weight per the doc: 1.5 at 24px, 1 at 16px.** Lighter than the current default of 2 — reads more refined, matches the muted palette philosophy. Overrides Lucide's default.
4. **Outline by default; filled only for active/selected state.** Introduces a new affordance Scribe doesn't currently use — bookmark/pin/star icons can switch outline → filled to indicate active state. Reserve filled style strictly for this.
5. **No emoji in chrome.** Replace the 3 files' emoji usage in UI labels/buttons/badges/empty states with Lucide equivalents. Emoji remain fine in *user content* (note bodies, chat messages the user typed).
Work cost: ~30-60 individual icon swaps across the 16 files. Mechanical; doesn't require redesign of any component.
### Voice and tone — adopt principles, defer formal audit
The doc's voice register applies to Scribe (understated mythic — plain for functional UI, flavored for empty/error/loading states). No formal sweep of every UI string yet.
**Approach:** apply the voice opportunistically as components are touched in the polish pass — when redesigning a settings tab, an empty state, or an error toast, rewrite the copy at the same time using the doc's register and examples table as the guide. A standalone audit pass is deferred unless drift becomes visible.
### Border philosophy — structural, not decorative
The doc treats borders as *structural* (Pewter neutral hairlines that say "boundary"), not decorative (Scribe today uses indigo-tinted borders that say "branded edge"). That principle suggests removing borders in places where surface tint and spacing already communicate separation.
**Borders to remove:**
- List rows (NotesListView, TasksListView, conversation history) — surface contrast + spacing should separate rows; current border reads as "boxed-in"
- Inline `ToolCallCard` inside chat bubbles — the bubble is already a container; an extra border feels like double-wrapping
- Filter chips and search-bar pills with a background tint — background does the work
- Empty-state callouts with dashed/bordered "nothing here yet" boxes — tinted background reads cleaner
**Borders to keep (genuinely structural):**
- Standalone card containers (Notes viewer, Task viewer, the new daily prep card)
- Modal / dialog edges
- Code blocks (separates content type, not just space)
- Focus rings (accessibility)
- Major section dividers within a panel
Border weight is not load-bearing for Scribe — happy to use the doc's 0.5px hairline default; the *placement* discipline matters more than the weight.
## Open threads (next iterations)
### Foundation pass — shipped 2026-04-27 (`7a9a8b7`)
Mechanical token + font + light-mode rewrite of `frontend/src/assets/theme.css`, plus a sweep of hardcoded indigo and `--color-accent-warm` references across ~14 component files. Action tokens (`--color-action-primary` Moss, `--color-action-secondary` Bronze, `--color-action-destructive` Oxblood, `--color-action-ghost-border` Pewter) are defined but not yet applied — buttons still flow through `--color-primary` and read as dusty-violet gradients in the meantime, by design. Spec lives at `docs/superpowers/specs/2026-04-27-design-system-polish-foundation-design.md` (gitignored, local-only).
### Surface phase — shipped 2026-04-27 (`93a3beb` → `3c1ec40`)
Bundled as Hybrid (option C from the brainstorm): Lucide cross-cutting first, then surface-by-surface for the judgment work. Spec lives at `docs/superpowers/specs/2026-04-27-design-system-polish-surface-design.md` (gitignored, local-only). Seven PRs landed on `dev`:
| PR | Commit | Surface | Notes |
|---|---|---|---|
| 1 | `93a3beb` | Lucide cross-cutting | 60 hand-inlined SVGs across 15 files → `lucide-vue-next`. Every chrome icon at 16 or 24. Emoji-as-icons (`✕`, `✓`, `🎤`, `📎`, `↻`, `↑`, `&times;`, `☀`/`☾`) swept across the chrome. AppLogo wordmark and the GraphView D3 mount kept as the legitimate exceptions. |
| 2 | `3d916d7` | Journal | Buttons audited; `↻` weather refresh → Lucide `RotateCcw`; assistant bubble line-height bumped; redundant `.journal-title` font-family dropped; dead `.news-section` CSS removed. |
| 3 | `4192a64` | Chat | `.message-content` long-form 1.7 line-height on assistant bubbles; ToolCallCard outer border removed (bubble already contains it; error state moved to a left-edge accent); bulk-delete recolored to Oxblood with Trash2 icon; `.bulk-link` accent → muted text. |
| 4 | `efb3534` | Knowledge cluster | `.prose` line-height bumped to 1.7 globally so all reading surfaces inherit. Save → Moss; Delete → Oxblood + Trash2; Edit / Advance → Moss; Convert / Share → Bronze. |
| 5 | `ff498ce` | Project + Workspace | Project save panel → Moss; milestone confirm/cancel → Moss/Bronze; modal-btn-danger and per-row delete affordances → Oxblood; "Open Workspace" stays accent (project-surface brand moment). |
| 6 | `541e2ed` | Settings | Densest button surface — every `.btn-save`, `.btn-primary`, `.btn-secondary`, `.btn-danger`, `.btn-danger-outline`, `.btn-toggle-open` reclassified per Hybrid; missing `.btn-danger` style block added (was unstyled before). |
| 7 | `3c1ec40` | Edge surfaces | Calendar `New Event` → Moss; EventSlideOver Save/Cancel/Delete reclassified; HomeView hero CTA stays accent (brand moment); two-weights-only sweep across Header/Home/Calendar/Graph. |
**Cross-cutting changes folded in as the work touched files:**
- Long-form 1.7 line-height on `.prose` (PR 4) — applies to Note viewer, Task viewer, chat assistant bubbles, anywhere markdown renders into a reading surface.
- Two-weights-only (400 + 500) — every `font-weight: 600` and `700` snapped to `500` across all surface PRs.
- Hardcoded `--color-danger` in destructive button contexts → `--color-action-destructive` (Oxblood). `--color-danger` (Error terracotta) preserved for validation/error messages, per the doc's distinction between Error and Destructive.
- Adjacent `&times;` / `✕` / unicode-arrow emoji swept opportunistically as files were touched (PRs 5, 7).
### Out of scope — deferred indefinitely
Items deliberately not addressed in this round; revisit when a real need surfaces:
- Lucide stroke-weight overrides (doc spec: 1.5 at 24, 1 at 16; current: Lucide default 2). Touched components if they read too heavy in practice.
- Filled-as-active icon state — no current affordance uses it; introduce when bookmark/pin/star toggles are added.
- Type-scale / spacing-scale CSS variables — components keep literal values.
- Token rename to `--fs-*` namespace — Scribe is the only FabledSword app sharing this codebase.
- FabledSword lockup placement — waiting on the actual heraldic mark to be drawn.
- Standalone voice/tone audit across every UI string — opportunistic-only; full sweep deferred unless drift becomes visible.
- A handful of editor utility buttons (`.btn-suggest-tags`, `.btn-link-all`, AI assist generate/proofread/accept/reject set, etc.) — currently ghost-styled and visually compliant; revisited only if they read off in practice.
### Open threads
*New threads will accumulate here as gaps surface in real use.*
+140 -48
View File
@@ -1,87 +1,179 @@
# Development
## Running it
## Workflow
All development is Docker-based. Do not install Python or Node dependencies locally.
```bash
docker compose -f docker-compose.dev.yml up --build # app + PostgreSQL at http://localhost:5000
docker compose -f docker-compose.dev.yml up --build app # rebuild after a change
docker compose -f docker-compose.dev.yml down -v # stop and wipe the database
```
# Start the full stack (app + PostgreSQL + Ollama)
docker compose up --build
The image builds the frontend, so a frontend change needs a rebuild too. For quicker frontend work, run Vite against the running app; it proxies `/api` to port 5000:
# Rebuild after backend changes (frontend changes require rebuild too)
docker compose up --build app
```bash
cd frontend && npm ci && npm run dev # http://localhost:5173
```
# Reset everything (wipes database)
docker compose down -v && docker compose up --build
## Checks
# Run checks (lint, format, typecheck, tests)
make check
CI runs these on every push; locally they need `uv sync --extra dev` and `cd frontend && npm ci`.
```bash
# Individual checks
make lint # ruff check src/
make fmt # ruff format src/
make typecheck # vue-tsc --noEmit
make test # pytest tests/
make check # all three
```
Unit tests run without a database. Tests marked `integration` need a real PostgreSQL and run in CI's integration job, against a schema built by the real migrations.
## Frontend Hot Reload
## Database migrations
The container runs `alembic upgrade head` on every start, after waiting for the database to accept connections.
The Docker setup does not include Vite's hot-reload dev server. After frontend changes, rebuild the image. For faster iteration during active frontend work, you can run Vite locally:
```bash
docker compose -f docker-compose.dev.yml exec app alembic revision -m "short_description"
cd frontend
npm install
npm run dev # Vite dev server at http://localhost:5173
```
- Number revisions sequentially (`0123_...`). Revision ids must fit in 32 characters, the width of `alembic_version.version_num`.
- Write both `upgrade()` and `downgrade()`.
- Adding a value to a `CHECK`-constrained column means dropping and re-adding the constraint in the same migration.
- Keep migrations to schema changes. Filling a derived table from a large one belongs in a background backfill after startup, where its cost doesn't hold up the boot.
- A migration that depends on application code (a key derivation, a serialiser) copies what it needs rather than importing it, so a later edit can't change what an old migration does.
Point the Vite dev server at the backend by setting `VITE_API_BASE_URL=http://localhost:5000` (or configure `vite.config.ts` proxy).
## The plugin
## Database Migrations
`plugin/` ships straight from git: installs fetch it through `.claude-plugin/marketplace.json`, and there is no build step. After changing anything under `plugin/` or `.claude-plugin/`, run `make mint-plugin` before committing. Claude Code only refreshes an installed plugin when its version changes, and CI fails a plugin change that didn't bump it.
Alembic migrations run automatically on container startup (`alembic upgrade head` in the `CMD`).
The hooks must run on a bare machine: POSIX shell, awk, sed and the usual coreutils, plus `git` and `curl`. They fail open, so a slow or unreachable Grimoire never blocks Claude Code.
To create a new migration:
```bash
# Inside the running app container
docker compose exec app alembic revision -m "description_of_change"
# Edit the generated file in alembic/versions/
```
## CI and releases
Migration conventions:
- Use raw SQL with `IF NOT EXISTS` guards for idempotency
- Use `DO $$ BEGIN CREATE TYPE … EXCEPTION WHEN duplicate_object THEN NULL; END $$` for enum types
- Number migrations sequentially (e.g. `0027_add_something.py`)
- Always provide both `upgrade()` and `downgrade()`
CI runs on Gitea Actions (workflow in `.forgejo/workflows/ci.yml`) in the shared `ci-python:3.14` image ([`ci-requirements.md`](../ci-requirements.md) lists what it relies on). Six jobs: TypeScript typecheck, plugin hooks, Python lint (plus design-token and dangling-style checks), unit tests, integration tests, and the image build, which waits on all of them except the plugin job.
## CI/CD
| Push | Image tags |
|------|-----------|
| `dev` | `:dev`, `:<sha>` |
| `main` | `:latest`, `:<sha>` |
| a `v*` tag | `:latest`, `:<version>`, `:<sha>` |
### Pipeline
`:<sha>` never moves, so it is the rollback point. Images publish to `git.fabledsword.com/bvandeusen/fabledgrimoire`, with a registry layer cache at `:cache`. The build needs the `REGISTRY_USER` and `REGISTRY_TOKEN` (a token with `write:packages`) repository secrets.
CI runs on Forgejo Actions, consuming the shared
[`ci-python:3.14`](https://git.fabledsword.com/bvandeusen/CI-runner) image
via `container.image` (Python 3.14 + Node 24 + ruff + uv + Docker CLI):
Work happens on `dev`. A release merges `dev` into `main` with a plain merge commit; a dated `vYYYY.MM.DD.HHMM` tag, matching the version the image reports, can mark it.
| Trigger | Jobs | Docker tags pushed |
|---------|------|--------------------|
| Push to `dev` | typecheck + lint + test → build | `:dev`, `:<sha>` |
| Tag `v*` on `main` | typecheck + lint + test → build | `:latest`, `:<version>`, `:<sha>` |
| Push to `main` | typecheck + lint + test | (no build) |
## Conventions
### Release Process
1. Work on `dev` branch — CI validates on every push
2. When ready, open a PR from `dev` → `main` in Forgejo
3. Merge the PR
4. Create a release via the Forgejo UI on `main` with a `v*` tag (e.g. `v26.03.23.1` — CalVer: `YY.MM.DD.N`)
5. The tag push triggers CI → build job pushes `:latest` + `:<version>` Docker images
6. After merging to main, sync dev back:
```bash
git checkout dev && git merge main && git push origin dev
```
### Runner
CI jobs schedule against the `python-ci` runner label and run inside the
shared `git.fabledsword.com/bvandeusen/ci-python:3.14` image (see
`ci-requirements.md` for what this project relies on from the image).
The runner deployment lives outside this repo; image bumps happen in
[CI-Runner](https://git.fabledsword.com/bvandeusen/CI-runner) via Renovate.
`infra/runner-compose.yml` + `infra/act-runner-config.yml` document the
runner-host deployment shape; the source of truth is the deployed
config on the runner host.
### Docker Registry
Images pushed to: `git.fabledsword.com/bvandeusen/scribe`
Cache tag: `:cache` (reduces build time ~80%)
Required secrets (repo → Settings → Secrets → Actions):
- `REGISTRY_USER` — Forgejo username
- `REGISTRY_TOKEN` — Forgejo PAT with `write:packages` scope
## Migration Chain
Current migration sequence (all idempotent raw SQL):
```
0001 create_notes_table
0002 create_tasks_table
0003 task_note_companion (data migration)
0004 merge_tasks_into_notes
0005 add_chat_tables
0006 add_settings_table
0007 add_title_and_updated_at_indexes
0008 add_users_and_user_id
0009 add_message_status
0010 add_app_logs_table
0011 add_password_reset_tokens
0012 add_invitation_tokens
0013 add_tool_calls_to_messages
0014 add_note_embeddings
0015 add_oauth_fields
0016 add_image_cache
0017 add_projects
0018 add_push_subscriptions
0019 add_events (dead code — internal CalDAV/Radicale table; Radicale was removed)
0020 add_milestones
0021 add_task_logs
0022 add_note_versions_and_drafts
0023 add_tags_to_note_versions
0024 add_session_version
0025 add_sharing_and_notifications
0026 add_briefing_tables
0027 add_api_keys
```
**Important:** Do NOT use `op.create_table()` or `sa.Enum()` — SQLAlchemy's event system can fire `CREATE TYPE` even with `create_type=False`, causing failures on re-run. Always use raw SQL with `IF NOT EXISTS` / `DO $$ BEGIN ... EXCEPTION WHEN duplicate_object` guards.
## Project Conventions
### Backend
- Services open their own session: `async with async_session() as session:` (from `grimoire.models`).
- Routes and MCP tools stay thin; the logic lives in `services/`, shared by both.
- Access checks go through `services/access.py`, never inline ownership checks.
- Every wait on something outside the process has a deadline.
- Services: `async with async_session() as session:` — import from `scribe.models`
- No `scribe.database` module
- Blueprint per resource: `routes/notes.py`, `routes/tasks.py`, etc.
- All business logic in `services/`; routes are thin wrappers
- Permission checks via `services/access.py` — never inline ownership checks in routes
### Frontend
- API calls go through the typed helpers in `frontend/src/api/client.ts`.
- Pinia stores for shared state, `ref()` for state local to a component; composables for reusable behaviour.
- Colours, spacing and type come from the design system's tokens in `theme.css`, which is generated, so CI rejects an unknown `var()`.
- API calls via `frontend/src/api/client.ts` typed helpers (`apiGet`, `apiPost`, `apiPatch`, `apiDelete`)
- Pinia stores for shared state; local `ref()` for component-only state
- Composables in `composables/` for reusable behaviour (autosave, keyboard nav, tag suggestions, …)
- Views are page-level components in `views/`; reusable UI in `components/`
### Commits
### Commit Style
```
type(scope): what the change does
type(scope): short description
Why, and anything a reader tracing this change later needs.
Longer body if needed.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
```
Types: `feat`, `fix`, `refactor`, `docs`, `chore`, `test`. Comments in the code describe how it works today; the history of a change belongs in its commit message.
Types: `feat`, `fix`, `refactor`, `docs`, `chore`, `test`
Scopes: feature area (e.g. `chat`, `journal`, `mcp`, `notes`)
## Testing
```bash
# Run all tests
make test
# Run specific test file
docker compose exec app /opt/venv/bin/pytest tests/test_auth.py -v
```
Tests are in `tests/`. They run against a real PostgreSQL instance in CI (not mocked). Keep tests integration-style where possible — mock failures have historically masked real migration bugs.
+120 -83
View File
@@ -1,131 +1,168 @@
# Features
Grimoire is a self-hosted system of record for the work you do with Claude Code. Claude records what it learned, decided and built, and that record reaches the next session, so a solution is recalled rather than reinvented. There is no language model inside Grimoire: Claude reaches it through the MCP endpoint and the Claude Code plugin, and the web UI is where people read and edit the same records.
Scribe is a self-hosted work system-of-record for software projects, built to be
driven by Claude Code. There is **no in-app LLM** — Claude is the sole assistant,
reaching Scribe through a built-in MCP endpoint and a bundled Claude Code plugin. The
web UI is a clean surface for humans to read and edit the same data.
## Notes
A Markdown editor (Tiptap) with headings, lists, code blocks and checklists inline, and a `/` menu for blocks.
Write in Markdown with a live-preview editor (Tiptap/ProseMirror). Headings, bold,
italic, lists, code blocks, and task checklists render inline. A slash-command menu
(`/`) inserts common blocks.
- **Wikilinks**: `[[Title]]` or `[[Title|text]]`, suggested as you type; backlinks show in the viewer.
- **Tags**, with autocomplete. Hierarchical tags (`area/backend`) match their parent's filter.
- **Version history**: browse, diff and restore earlier versions of a note.
- **Draft recovery**: unsaved edits survive a reload.
- **Wikilinks** — Link notes with `[[Title]]` or `[[Title|Display Text]]`. Clicking
navigates to (or auto-creates) the referenced note; the editor suggests existing
titles while you type `[[`. Backlinks appear in the note viewer sidebar.
- **Tags** — First-class `ARRAY[text]` column with autocomplete. Hierarchical tags
(`area/backend`) supported — filtering by `area` matches all `area/*` children.
- **Version history** — Every body edit snapshots a version (up to 20 per note).
Browse, diff, and restore from the editor's History panel.
- **Draft recovery** — In-progress edits persist across page loads and are restored
when you reopen a note.
- **Convert freely** — Turn a note into a task (sets `status=todo`) or back again.
## Tasks
## Tasks and Issues
A task is a note with a status (`todo`, `in_progress`, `done`, `cancelled`), plus priority, due date, milestone and parent task.
Tasks carry status (`todo` → `in_progress` → `done`/`cancelled`), priority
(`none`/`low`/`medium`/`high`), due date, milestone assignment, and a parent task
(sub-tasks). Notes and tasks share one model — a task is a note with a status.
- **Kinds**: `work`; `issue`, for corrective work, which can link the task it arose from; and `spike`, a time-boxed investigation whose output is an answer.
- **Work logs**: timestamped entries that record progress without rewriting the body. They are what the next session reads first.
- **Claims**: a session that logs on a task holds it, so after a context compaction the work comes back to that session, and other sessions can see who is on it.
- **Recurring tasks** spawn their next occurrence on a schedule.
- **Work logs** — Append timestamped progress entries (with optional duration) to a
task without rewriting its body; shown chronologically in the task view.
- **Issues** — A task whose `kind` is corrective: a problem you fixed or are fixing,
with symptom → root cause → fix in the body. An issue can link the task it arose
from and the System(s) it touches.
- **Recurring tasks** — An interval or calendar recurrence rule spawns the next
occurrence when a task is completed (a background job drains due spawns).
- **Sub-tasks** — Any task can have children via `parent_id`; the viewer shows them
inline.
## Projects, milestones and systems
## Projects and Milestones
- **Projects** carry a goal, a status and a colour, and group everything else. Entering a project gives Claude a short primer: its goal, recent plans and their next steps, open tasks, its systems and its design system.
- **Milestones are plans**: a body holding the design (goal, approach, verification) and ordered step tasks. Starting a plan that matches an active milestone hands back that milestone instead of opening a second one.
- **Systems** are a project's named areas ("auth", "retrieval"). Records are tagged with them, each has a charter saying what belongs there, and each maps to a canonical area shared across projects.
- **Duplicate gates**: creating a record that reads like an existing one returns the existing one to update instead.
- **Projects** — Group related notes and tasks. Title, description, goal, status
(`active`/`paused`/`completed`/`archived`), and a colour.
- **Milestones** — Ordered stages within a project. A milestone is also the home of a
**plan** — its body holds the design (Goal/Approach/Verification) and its child
tasks are the steps. Completion percentage is shown on the project page.
- **Kanban view** — `/projects/:id` groups tasks by milestone in a column layout with
status-advance buttons on the cards.
## Rules, lessons and moments
## Systems
- **Rulebooks → topics → rules.** A rule in a rulebook applies in every project; a project rule applies only there. A project that departs from a global rule says so with an `overrides` relation.
- **Rules arrive by retrieval.** Nothing is preloaded: a rule reaches a session when what Claude is about to do matches its trigger (`when_to_apply`), whether that's the operator's message, a command or a file being written.
- **Moments**: a rule about *when* something is done (finishing a task, reporting back, pushing) can be mounted on a moment of work, and arrives whenever that moment happens, whatever the words look like. Settings → General shows which actions reach each moment.
- **Versions and verification**: every edit to a rule keeps the text it replaced. A rule that asserts a fact about someone else's software can say how to check it and when it stops being true, and a sweep lists rules due for a check.
- **Preferences** are rules with the force of guidance rather than obligation.
- **Lessons** record something learned the hard way, keyed to the situation where it applies, and link to the rules they are instances of.
- **Reply shapes**: the default shape of Claude's replies (completion report, plan for review, question) arrives at the moment it applies. Settings → General shows them.
A **System** is a per-project, reusable, self-describing subsystem or area (e.g.
"auth", "billing"). Associate any note, task, or issue with a System so research,
build-work, and fixes for the same area line up and recurring problem-spots surface.
## Snippets and the shape ledger
## Rules and Rulebooks
- **Snippets** are recorded prior art: a reusable piece of code with when to reach for it, its signature and its canonical location in a repo. Claude searches them before building and is shown matching ones before it writes a file.
- **Forge integration**: with a git forge connected, Grimoire checks a snippet's cached body against its source and marks it current or diverged. Push webhooks keep that fresh.
- **The shape ledger**: Grimoire extracts the code shapes from a bound repo and tracks how each is accounted for: an instance of a snippet, a deliberate variant with its reason, or a one-off. Coverage shows what is still unclassified.
Scribe stores the operator's engineering and workflow **rules** so Claude follows them
across sessions.
## Family canon
- **Rulebooks → topics → rules** — Rules are grouped by topic inside a rulebook.
- **Always-on rules** — A rulebook can be flagged always-on; its rules load at the
start of every session through the plugin's push channel.
- **Per-project scope** — A project subscribes to rulebooks, and can add
project-scoped rules or suppress individual inherited rules/topics.
Projects on the same platform (Android apps, Go services, ...) share good ideas. An idea promoted to the family's canon reaches every project on that platform, and each answers it: adopted, a variant with its reason, or owed (filed as a task). `/family` shows the ideas and every project's answers.
## Stored Processes
## Design systems
Reusable saved prompts (a note with `note_type=process`) — e.g. a drift-audit or a
DRY pass. The bundled plugin syncs each Process into a local Claude Code skill stub
(`/scribe:sync`) that auto-surfaces by relevance and fetches the live procedure on
demand.
Tokens (colour, type, space, motion, ...) and written guidance, with inheritance: a project's design system can extend a house style and record where it departs. Grimoire generates a stylesheet from it, checks code snippets against it, and gives Claude the values to use before it writes UI. `/design-systems` edits them.
## Search and Knowledge Injection
## Stored processes
- **Semantic search** — pgvector-backed similarity search over notes and tasks
(in-process `fastembed` embeddings; no external model).
- **Proactive knowledge-injection** — the plugin's `UserPromptSubmit` hook surfaces a
short, high-confidence menu of maybe-relevant note *titles* into Claude's context
each turn; Claude pulls a full body only when it judges it relevant. Gated so it
stays quiet on most turns and never repeats within a session.
Saved procedures, such as a drift audit or a DRY pass. The plugin syncs each one into a Claude Code skill (`/grimoire:sync`), which fetches the live procedure when it is invoked.
## Knowledge Graph
## Search and context injection
- **Semantic search** over notes, tasks, rules, milestones and systems, with embeddings computed in-process (no external model service). It falls back to keyword search if the embedder is unavailable.
- **Context injection**: the plugin's hooks show Claude a short menu of possibly relevant records at the moments they matter: rules and prior art when a prompt arrives, before a command or a file write, and when a moment of work is reached. Claude opens a full record only when it judges it relevant.
- **Tuning**: each surface has its own thresholds and budget, in Settings → General. Every change is logged with its reason, and retrieval telemetry shows what was surfaced and what was opened.
`/graph` renders notes, tasks, and tags as a D3 force-directed graph. Tag nodes
cluster notes that share tags; invisible project hubs attract project members. Physics
controls (repulsion, link distance/strength, hub pull, gravity); click a node to peek,
click a tag to filter.
## Claude via MCP and the plugin
The whole store is reachable through the MCP endpoint at `/mcp`, authenticated with an API key. The Claude Code plugin in this repo adds:
The whole store is reachable by Claude through a built-in **MCP endpoint at `/mcp`**
(Bearer-auth with an API key). The **Scribe Claude Code plugin** (shipped in this
repo) wires it up:
- hooks for session start, each prompt, before and after tool calls, the end of a turn, compaction and session end, all failing open if Grimoire is unreachable;
- skills for the working practices: using Grimoire, writing plans, systematic debugging, verification, brainstorming, reporting back, reusing code, shape accounting and family canon;
- your stored processes as skills.
- a `SessionStart` hook that injects the operator's always-on rules + active-project
context so Scribe surfaces without being asked (fail-open if Scribe is unreachable);
- universal process-skills — writing-plans, systematic-debugging, verification,
brainstorming — that route their output into Scribe;
- your saved Processes auto-surfaced as skills.
See [API Keys & MCP](api-keys-and-mcp.md).
## The web UI
## Sharing and Collaboration
- **Dashboard**: what to work on next. **Knowledge** browses everything.
- **Graph** (`/graph`): notes, tasks and tags as a force-directed graph, with physics controls and filtering by tag.
- **Trash**: deleted records wait here until retention purges them, and can be restored.
- **Installable** as a desktop or mobile app.
- **Share** — Share any project, note, or task with users or groups at
`viewer`/`editor`/`admin` levels from the viewer/project toolbar.
- **Groups** — Admins create platform-wide groups and assign `member`/`owner` roles;
share a resource with a group in one action.
- **Shared with me** — `/shared` lists incoming shares with permission badges.
- **Notifications** — An in-app bell (unread count, polled) fires when a project or
note is shared with you or you're added to a group.
## Sharing
Every read and mutation is scoped by owner + direct shares + group shares.
Share a project, note or task with users or groups, as viewer, editor or admin. Admins manage groups. `/shared` lists what has been shared with you, and an in-app bell notifies you of new shares. Every read and write is checked against ownership and shares.
## Data Export and Backup
## Export and backup
- **Personal export** — download all your notes/tasks as a Markdown ZIP (YAML
frontmatter) or a JSON array.
- **Admin backup** — full application backup/restore (format v4) with ID remapping on
restore for cross-instance migration.
- **Personal export**: your notes and tasks as a Markdown ZIP (with YAML frontmatter) or JSON.
- **Admin backup and restore**, with ids remapped so a backup restores onto another instance.
- **Scheduled backups**: a daily, dated, uncompressed tar to a folder or an S3-compatible bucket, laid out so a deduplicating tool such as restic stores only what changed. `python -m grimoire.restore` brings a fresh instance back from one. See [Backups](backups.md).
## Progressive Web App
## Sign-in
Installable as a desktop or mobile app; a service worker caches the shell.
Email and password, plus optional OIDC single sign-on (Authentik, Keycloak, ...) that links to a matching local account. Invite links, password reset and signing out other sessions are included. See [SSO / OAuth](sso-oauth.md).
## Authentication
Native email + password, plus optional **OIDC** sign-in (Authentik, Keycloak, etc.)
that links to a matching local account. Invite links, a registration toggle, password
reset, and session invalidation are included. See [SSO / OAuth](sso-oauth.md).
## Settings
| Tab | Contents |
|-----|----------|
| General | Timezone, trash retention, retrieval thresholds, moments, reply shapes |
| Account | Email, password, active sessions |
| Notifications | Email notifications |
| Integrations | SearXNG, git forge connections |
| Data | Export, backup and restore, scheduled backups (admin) |
| MCP Access | API keys and the command to connect Claude |
| Config (admin) | What's running, application URL, plugin marketplace, database maintenance, SMTP, forge webhook |
| Shared areas (admin) | The canonical areas systems map to |
| Platforms (admin) | The platform catalogue family canon is grouped by |
| Users (admin) | Registration, invites, users |
| Logs (admin) | Error, audit and usage logs |
| Groups (admin) | Groups and membership |
| General | Instance preferences (key/value) |
| Account | Email change, password change, session invalidation |
| Profile | Per-user profile fields |
| Integrations | SearXNG status |
| Data | Personal export; backup / restore (admin) |
| API Keys | Create/revoke `fmcp_` keys for the MCP endpoint |
| Config (admin) | Base URL, SMTP, DB-maintenance schedule |
| Users (admin) | User list, invite links, registration toggle |
| Logs (admin) | Error, audit, and usage logs with search |
| Groups (admin) | Create/manage groups and membership |
## Keyboard shortcuts
## Keyboard Shortcuts
| Key | Action |
|-----|--------|
| `g` `h` | Dashboard |
| `g` `n` | Notes |
| `g` `t` | Knowledge (everything, tasks included) |
| `g` `p` | Projects |
| `g` `r` | Rules |
| `g` `g` | Graph |
| `g` `x` | Trash |
| `g` + `h` | Home (dashboard) |
| `g` + `n` | Notes |
| `g` + `t` | Knowledge (tasks) |
| `g` + `p` | Projects |
| `g` + `r` | Rulebooks |
| `g` (bare) | Graph |
| `g` + `x` | Trash |
| `n` | New note |
| `t` | New task |
| `e` | Edit the note you're viewing |
| `/` | Focus search |
| `?` | Show shortcuts |
| `j` / `k` | Move through a list |
| `Enter` | Open the selected item |
| `Esc` | Close the overlay, leave the field, then go to the dashboard |
| `Ctrl+S` | Save in the editor |
| `?` | Show shortcuts panel |
| `j` / `k` | Navigate list items |
| `Enter` | Open selected item |
| `e` | Edit current item |
| `Esc` | Close panel / blur / go home (progressive) |
| `Ctrl+S` | Save in editor |
+5 -5
View File
@@ -1,6 +1,6 @@
# OAuth / OIDC SSO Setup
Grimoire supports single sign-on via any OpenID Connect provider.
Fabled Assistant supports single sign-on via any OpenID Connect provider.
This guide covers Authentik, but the same pattern works with Keycloak, Authelia, Zitadel, etc.
## 1. Create the provider in Authentik
@@ -8,7 +8,7 @@ This guide covers Authentik, but the same pattern works with Keycloak, Authelia,
1. Log in to the Authentik admin UI.
2. Go to **Applications → Providers → Create → OAuth2/OpenID Provider**.
3. Fill in:
- **Name:** `Grimoire` (or whatever you like)
- **Name:** `Fabled Assistant` (or whatever you like)
- **Authorization flow:** your default authorization flow
- **Client type:** `Confidential`
- **Redirect URIs:** `https://your-fabled-domain/api/auth/oauth/callback`
@@ -20,7 +20,7 @@ This guide covers Authentik, but the same pattern works with Keycloak, Authelia,
https://auth.example.com/application/o/fabled-assistant/
```
## 2. Configure Grimoire
## 2. Configure Fabled Assistant
Add the following environment variables to the `app` service in `docker-compose.yml`:
@@ -64,7 +64,7 @@ docker compose up --build -d
## 4. Account Linking
When a user logs in via OAuth for the first time, Grimoire checks in this order:
When a user logs in via OAuth for the first time, Fabled checks in this order:
1. **Existing OAuth sub** — returns that user immediately.
2. **Matching email** — if a local account already exists with the same email, the OAuth identity is linked to it automatically. The user retains all their notes and tasks.
@@ -92,4 +92,4 @@ The backend will reject any `POST /api/auth/login` or `POST /api/auth/register`
| Zitadel | `https://your-instance.zitadel.cloud` |
| Google | `https://accounts.google.com` |
The OIDC discovery endpoint (`<issuer>/.well-known/openid-configuration`) must be publicly reachable from the Grimoire container (server-to-server call at login time).
The OIDC discovery endpoint (`<issuer>/.well-known/openid-configuration`) must be publicly reachable from the Fabled container (server-to-server call at login time).
+3 -4
View File
@@ -4,14 +4,13 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
<link rel="manifest" href="/manifest.json">
<meta name="theme-color" content="#14171A">
<meta name="theme-color" content="#6c63ff">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
<meta name="apple-mobile-web-app-title" content="Grimoire">
<title>Fabled Grimoire</title>
<meta name="apple-mobile-web-app-title" content="Scribe">
<title>Fabled Scribe</title>
</head>
<body>
<div id="app"></div>
+2 -3
View File
@@ -1,13 +1,12 @@
{
"name": "grimoire-frontend",
"name": "scribe-frontend",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "grimoire-frontend",
"name": "scribe-frontend",
"version": "0.1.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@fullcalendar/core": "^6.1.20",
"@fullcalendar/daygrid": "^6.1.20",
+1 -2
View File
@@ -1,7 +1,6 @@
{
"name": "grimoire-frontend",
"name": "scribe-frontend",
"private": true,
"license": "AGPL-3.0-only",
"version": "0.1.0",
"type": "module",
"scripts": {
Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.4 KiB

+31 -16
View File
@@ -1,18 +1,33 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
<!-- page block, seen at the fore-edge -->
<path d="M8.5 4.5 H25.5 Q27 4.5 27 6 V27 Q27 28.5 25.5 28.5 H8.5 Z" fill="#E9DFC6"/>
<path d="M25 7 V26.5" stroke="#C9BB97" stroke-width="0.6"/>
<!-- cover -->
<path d="M5 4 Q5 3 6 3 H24 Q25.5 3 25.5 4.5 V26.5 Q25.5 28 24 28 H6 Q5 28 5 27 Z" fill="#5B4A8A"/>
<!-- spine band -->
<rect x="5" y="3" width="3.2" height="25" rx="0.6" fill="#47396E"/>
<!-- corner fittings -->
<path d="M21.5 3 H24 Q25.5 3 25.5 4.5 V7 Z" fill="#D4A94E"/>
<path d="M21.5 28 H24 Q25.5 28 25.5 26.5 V24 Z" fill="#D4A94E"/>
<!-- clasp across the fore-edge -->
<rect x="22.5" y="14" width="5.5" height="3.6" rx="0.8" fill="#D4A94E"/>
<circle cx="26.4" cy="15.8" r="0.7" fill="#8A6A22"/>
<!-- sigil: a four-pointed star in a ring -->
<circle cx="15.5" cy="15.5" r="5.2" fill="none" stroke="#D4A94E" stroke-width="0.9"/>
<path d="M15.5 11.4 L16.5 14.5 L19.6 15.5 L16.5 16.5 L15.5 19.6 L14.5 16.5 L11.4 15.5 L14.5 14.5 Z" fill="#F1D48A"/>
<style>
.book { fill: #6366f1; stroke: #4f46e5; }
.spine { stroke: #4f46e5; }
.lines { stroke: #a5b4fc; }
@media (prefers-color-scheme: dark) {
.book { fill: #e2e8f0; stroke: #cbd5e0; }
.spine { stroke: #cbd5e0; }
.lines { stroke: #4a5568; }
}
</style>
<!-- Book body -->
<path class="book" d="M4 7 C4 7 8 5 16 6 C24 5 28 7 28 7 L28 26 C28 26 24 24 16 25 C8 24 4 26 4 26 Z" stroke-width="0.5"/>
<!-- Book spine -->
<line class="spine" x1="16" y1="6" x2="16" y2="25" stroke-width="0.8"/>
<!-- Left page lines -->
<line class="lines" x1="7" y1="11" x2="14" y2="10.5" stroke-width="0.6" stroke-linecap="round"/>
<line class="lines" x1="7" y1="14" x2="14" y2="13.5" stroke-width="0.6" stroke-linecap="round"/>
<line class="lines" x1="7" y1="17" x2="14" y2="16.5" stroke-width="0.6" stroke-linecap="round"/>
<line class="lines" x1="7" y1="20" x2="12" y2="19.5" stroke-width="0.6" stroke-linecap="round"/>
<!-- Right page lines -->
<line class="lines" x1="18" y1="10.5" x2="25" y2="11" stroke-width="0.6" stroke-linecap="round"/>
<line class="lines" x1="18" y1="13.5" x2="25" y2="14" stroke-width="0.6" stroke-linecap="round"/>
<line class="lines" x1="18" y1="16.5" x2="25" y2="17" stroke-width="0.6" stroke-linecap="round"/>
<!-- Magic spark -->
<g transform="translate(24, 5)">
<path d="M0 -4 L1 -1 L4 0 L1 1 L0 4 L-1 1 L-4 0 L-1 -1 Z" fill="#f6ad55"/>
<path d="M0 -2.5 L0.6 -0.6 L2.5 0 L0.6 0.6 L0 2.5 L-0.6 0.6 L-2.5 0 L-0.6 -0.6 Z" fill="#fbd38d"/>
</g>
<!-- Secondary sparkles -->
<circle cx="20" cy="3" r="0.7" fill="#f6ad55" opacity="0.7"/>
<circle cx="27" cy="8" r="0.5" fill="#fbd38d" opacity="0.6"/>
</svg>

Before

Width:  |  Height:  |  Size: 1.0 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

+7 -17
View File
@@ -1,27 +1,17 @@
{
"name": "Fabled Grimoire",
"short_name": "Grimoire",
"description": "A self-hosted work system-of-record for software projects, built to be driven by Claude Code.",
"name": "Fabled Scribe",
"short_name": "Scribe",
"description": "Your self-hosted second brain with AI assistance",
"start_url": "/",
"display": "standalone",
"background_color": "#14171A",
"theme_color": "#14171A",
"background_color": "#1a1a2e",
"theme_color": "#6c63ff",
"orientation": "portrait-primary",
"icons": [
{
"src": "/favicon.svg",
"src": "/favicon.ico",
"sizes": "any",
"type": "image/svg+xml"
},
{
"src": "/icon-192.png",
"sizes": "192x192",
"type": "image/png"
},
{
"src": "/icon-512.png",
"sizes": "512x512",
"type": "image/png"
"type": "image/x-icon"
}
]
}
+26 -50
View File
@@ -7,19 +7,12 @@ import { useTheme } from "@/composables/useTheme";
import { useShortcuts } from "@/composables/useShortcuts";
import { useAuthStore } from "@/stores/auth";
import { useSettingsStore } from "@/stores/settings";
import { apiPut } from "@/api/client";
import { fetchVersion } from "@/api/version";
import { apiGet, apiPut } from "@/api/client";
useTheme();
const router = useRouter();
// THREE states, not two. `null` is "not answered yet" and renders nothing; a
// string renders; `appVersionFailed` renders its own thing. Defaulting to the
// literal "dev" and swallowing the error would make an instance that cannot
// answer indistinguishable from a local build that genuinely reports "dev" — a blank standing in for `unknown`, in the one
// readout whose whole job is to say what is running.
const appVersion = ref<string | null>(null);
const appVersionFailed = ref(false);
const appVersion = ref("dev");
const authStore = useAuthStore();
const settingsStore = useSettingsStore();
const { showShortcuts, toggleShortcuts, closeShortcuts } = useShortcuts();
@@ -84,7 +77,7 @@ function onGlobalKeydown(e: KeyboardEvent) {
switch (e.key) {
case "h": router.push("/"); break;
case "n": router.push("/notes"); break;
case "t": router.push("/knowledge"); break;
case "t": router.push("/"); break;
case "p": router.push("/projects"); break;
case "r": router.push("/rules"); break;
case "g": router.push("/graph"); break;
@@ -126,12 +119,10 @@ onMounted(async () => {
startAppServices();
}
try {
appVersion.value = (await fetchVersion()).version;
const data = await apiGet<{ version: string }>("/api/version");
appVersion.value = data.version;
} catch {
// Not silent any more: the footer says it could not find out, rather than
// showing a version it never received. The full readout (version, channel,
// commit, build) lives in Settings → Config.
appVersionFailed.value = true;
// silent — version display is non-critical
}
});
@@ -160,10 +151,7 @@ onUnmounted(() => {
<div id="main-content" class="app-content">
<router-view />
</div>
<footer class="app-footer">
<span v-if="appVersion">v{{ appVersion }}</span>
<span v-else-if="appVersionFailed">version unknown</span>
</footer>
<footer class="app-footer">v{{ appVersion }}</footer>
</div>
<!-- Keyboard shortcuts overlay -->
@@ -181,7 +169,7 @@ onUnmounted(() => {
<kbd class="shortcut-key">g</kbd>
<span class="shortcut-key-sep">+</span>
<kbd class="shortcut-key">h</kbd>
<span class="shortcut-desc">Dashboard</span>
<span class="shortcut-desc">Home</span>
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">g</kbd>
@@ -201,18 +189,6 @@ onUnmounted(() => {
<kbd class="shortcut-key">p</kbd>
<span class="shortcut-desc">Projects</span>
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">g</kbd>
<span class="shortcut-key-sep">+</span>
<kbd class="shortcut-key">r</kbd>
<span class="shortcut-desc">Rules</span>
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">g</kbd>
<span class="shortcut-key-sep">+</span>
<kbd class="shortcut-key">g</kbd>
<span class="shortcut-desc">Graph</span>
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">g</kbd>
<span class="shortcut-key-sep">+</span>
@@ -221,7 +197,7 @@ onUnmounted(() => {
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">Esc</kbd>
<span class="shortcut-desc">Unfocus field → go to the dashboard</span>
<span class="shortcut-desc">Unfocus field → go home</span>
</div>
<div class="shortcut-row">
<kbd class="shortcut-key">?</kbd>
@@ -278,7 +254,7 @@ onUnmounted(() => {
left: 0.5rem;
z-index: 9999;
padding: 0.4rem 0.75rem;
background: var(--fs-accent);
background: var(--color-primary);
color: var(--fs-text-on-action);
border-radius: 0 0 4px 4px;
font-size: 0.875rem;
@@ -314,7 +290,7 @@ onUnmounted(() => {
text-align: center;
padding: 0.2rem 0;
font-size: 0.68rem;
color: var(--fs-text-tertiary);
color: var(--color-text-muted);
opacity: 0.45;
user-select: none;
letter-spacing: 0.03em;
@@ -324,17 +300,17 @@ onUnmounted(() => {
.shortcuts-overlay {
position: fixed;
inset: 0;
background: var(--fs-overlay);
background: var(--color-overlay, rgba(0, 0, 0, 0.45));
z-index: 9000;
display: flex;
align-items: center;
justify-content: center;
}
.shortcuts-panel {
background: var(--fs-surface-raised);
border: 1px solid var(--fs-border-color);
border-radius: var(--fs-radius-lg);
box-shadow: 0 8px 32px var(--color-shadow);
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: var(--radius-md, 8px);
box-shadow: 0 8px 32px var(--color-shadow, rgba(0,0,0,0.2));
width: min(420px, 92vw);
overflow: hidden;
}
@@ -343,25 +319,25 @@ onUnmounted(() => {
align-items: center;
justify-content: space-between;
padding: 0.85rem 1rem 0.75rem;
border-bottom: 1px solid var(--fs-border-color);
border-bottom: 1px solid var(--color-border);
}
.shortcuts-header h3 {
margin: 0;
font-size: 1rem;
font-weight: 600;
color: var(--fs-text-primary);
color: var(--color-text);
}
.shortcuts-close {
background: none;
border: none;
font-size: 1.4rem;
line-height: 1;
color: var(--fs-text-tertiary);
color: var(--color-text-muted);
cursor: pointer;
padding: 0 0.25rem;
}
.shortcuts-close:hover {
color: var(--fs-text-primary);
color: var(--color-text);
}
.shortcuts-body {
padding: 0.75rem 1rem 1rem;
@@ -374,7 +350,7 @@ onUnmounted(() => {
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.06em;
color: var(--fs-text-tertiary);
color: var(--color-text-muted);
margin-bottom: 0.4rem;
}
.shortcut-row {
@@ -389,23 +365,23 @@ onUnmounted(() => {
justify-content: center;
min-width: 1.8rem;
padding: 0.15rem 0.4rem;
background: var(--fs-surface-raised);
border: 1px solid var(--fs-border-color);
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-bottom-width: 2px;
border-radius: 4px;
font-size: 0.78rem;
font-family: ui-monospace, monospace;
color: var(--fs-text-primary);
color: var(--color-text);
white-space: nowrap;
user-select: none;
}
.shortcut-key-sep {
font-size: 0.78rem;
color: var(--fs-text-tertiary);
color: var(--color-text-muted);
}
.shortcut-desc {
font-size: 0.875rem;
color: var(--fs-text-primary);
color: var(--color-text);
margin-left: 0.25rem;
}
-73
View File
@@ -1,73 +0,0 @@
import { apiGet, apiPost, apiPut } from "@/api/client";
/**
* Scheduled backups — `/api/admin/backups`. One dated,
* uncompressed tar per run, delivered to a folder or an S3 bucket.
*/
export interface BackupConfig {
enabled: boolean;
/** UTC hour the daily backup runs at, 0–23. */
hour: number;
destination: "folder" | "s3";
folder_path: string;
s3_endpoint: string;
s3_bucket: string;
s3_access_key: string;
s3_region: string;
s3_verify_tls: boolean;
}
export interface BackupRun {
started_at: string;
finished_at: string;
trigger: "scheduled" | "manual";
status: "ok" | "failed";
destination?: string;
file?: string;
size_bytes?: number;
where?: string;
error?: string;
duration_s: number;
}
export interface BackupsOverview {
config: BackupConfig;
/** Never the secret itself: whether one is kept, and whether this
* instance can still open it. */
secret: { set: boolean; readable: boolean };
/** What stops the saved config from sending a backup, in words. */
problems: string[];
running: boolean;
/** Newest first, at most 30. */
runs: BackupRun[];
last_run: BackupRun | null;
last_success: BackupRun | null;
overdue: boolean;
}
/** What the form sends: config fields, plus the secret when one was typed. */
export type BackupChanges = Partial<BackupConfig> & {
s3_secret?: string;
clear_s3_secret?: boolean;
};
export function getBackups(): Promise<BackupsOverview> {
return apiGet<BackupsOverview>("/api/admin/backups");
}
export function saveBackups(changes: BackupChanges): Promise<BackupsOverview> {
return apiPut<BackupsOverview>("/api/admin/backups", changes);
}
/** Ask the destination whether it would take a backup, with the form's
* unsaved values. The server gives the destination 45s; this waits a little
* longer so its own answer, not the client's deadline, is what arrives. */
export function checkBackupDestination(
changes: BackupChanges,
): Promise<{ ok: boolean; message: string }> {
return apiPost<{ ok: boolean; message: string }>("/api/admin/backups/check", changes, { timeoutMs: 60_000 });
}
export function runBackupNow(): Promise<{ status: string }> {
return apiPost<{ status: string }>("/api/admin/backups/run", {});
}
-81
View File
@@ -1,81 +0,0 @@
/**
* Canonical systems — the GLOBAL area vocabulary every project's Systems can
* map onto.
*
* The mapping is an ASSOCIATION, never a rename: a project's System keeps the
* name the project gave it, and `canonical_id` only records which shared area
* it is an instance of. An unmapped System is fully usable — the catalog is a
* convergence aid, not a gate.
*/
import { apiGet, apiPost, apiPatch, apiPut } from "@/api/client";
export interface CanonicalSystem {
id: number;
name: string;
/** The match key: lowercase, "&" folded to "and", punctuation collapsed. */
slug: string;
description: string | null;
order_index: number;
created_at: string | null;
updated_at: string | null;
}
/**
* A suggested mapping. `basis` is the whole point of showing it:
* - `exact` — the names differ only in spelling. Mechanical.
* - `overlap` — they share a meaningful word. A judgment call the reviewer is
* making, and it must never be presented as if it were the first.
*/
export interface CanonicalMatch {
id: number;
name: string;
basis: "exact" | "overlap";
score?: number;
}
export interface MappingProposal {
system_id: number;
system_name: string;
canonical_id: number;
canonical_name: string;
basis: "exact" | "overlap";
score: number;
}
export async function listCanonicalSystems(): Promise<CanonicalSystem[]> {
const data = await apiGet<{ canonical_systems: CanonicalSystem[] }>(
"/api/canonical-systems",
);
return data.canonical_systems;
}
/** Admin only — a global list anyone can extend stops being shared. */
export async function createCanonicalSystem(data: {
name: string;
description?: string;
}): Promise<CanonicalSystem> {
return apiPost("/api/canonical-systems", data);
}
export async function updateCanonicalSystem(
id: number,
data: Partial<{ name: string; description: string; order_index: number }>,
): Promise<CanonicalSystem> {
return apiPatch(`/api/canonical-systems/${id}`, data);
}
/** Proposals for a project's UNMAPPED Systems. Reads only — nothing applied. */
export async function proposeMappings(projectId: number): Promise<MappingProposal[]> {
const data = await apiGet<{ proposals: MappingProposal[] }>(
`/api/projects/${projectId}/canonical-proposals`,
);
return data.proposals;
}
/** Apply or clear one mapping. `null` unmaps. */
export async function mapSystem(
systemId: number,
canonicalId: number | null,
): Promise<{ id: number; canonical_id: number | null }> {
return apiPut(`/api/systems/${systemId}/canonical`, { canonical_id: canonicalId });
}
+50 -140
View File
@@ -38,134 +38,41 @@ async function handleResponse<T>(res: Response, path: string): Promise<T> {
return res.json() as Promise<T>;
}
/**
* The server's `{"error": "..."}` message from a failed call, or `fallback`
* when the failure carried none (network error, non-JSON body). The one place
* the error envelope is unpacked on the client, so no view restates it as a
* `"body" in e` branch at its catch site.
*/
export function apiErrorMessage(e: unknown, fallback: string): string {
if (e && typeof e === "object" && "body" in e) {
const body = (e as { body?: { error?: unknown } }).body;
if (body && typeof body.error === "string" && body.error) return body.error;
}
return fallback;
}
/**
* How long an ordinary JSON call may wait before it is declared failed.
*
* A wait with no deadline is a bug. `fetch`'s own default is to wait
* as long as the browser will, which is not a deadline — it is the absence of
* one, and it renders as a spinner that never resolves. There is no state a
* surface can show for "pending forever" that is not a lie.
*
* 30s is chosen to be longer than anything healthy: it has to clear a cold
* embedding call and a list view under connection-pool contention, so
* tripping it means something is genuinely wrong rather than merely busy. Slow
* BY DESIGN is a different case and passes its own value — see the callers in
* SettingsView that do.
*/
const DEFAULT_TIMEOUT_MS = 30_000;
/** HTTP 408. Not a status any Grimoire route returns, so it unambiguously means
* "the client gave up" rather than anything the server said. */
const CLIENT_TIMEOUT_STATUS = 408;
/**
* How long a STREAM may take to answer with its headers.
*
* Streams are the one case a wall-clock deadline would break: a long-lived SSE
* connection is *supposed* to stay open, and `AbortSignal.timeout` would kill
* it mid-flight along with the body. But that does not exempt them from rule
* 156 — it relocates the deadline. Two different waits are involved:
*
* connect — the server answering with headers. CAN fail to answer, so it
* carries this deadline, cleared the moment headers arrive.
* stream — the body, open indefinitely on purpose. Its failure mode is
* going quiet, which a timeout cannot tell from being idle; that
* is what reconnection and Last-Event-ID are for, not this.
*
* Reading the connect as exempt because "the stream is long-lived" is the easy
* mistake here, and it leaves an unreachable server looking like a quiet one.
*/
const STREAM_CONNECT_TIMEOUT_MS = 15_000;
/**
* A signal that aborts if headers do not arrive in time, plus the `settle` to
* call once they do. After `settle()` the returned signal never fires, so the
* stream body runs unbounded — which is the intent.
*/
function connectDeadline(base: AbortSignal): { signal: AbortSignal; settle: () => void } {
const gate = new AbortController();
const timer = setTimeout(
() => gate.abort(new DOMException("stream did not connect in time", "TimeoutError")),
STREAM_CONNECT_TIMEOUT_MS,
);
return {
signal: AbortSignal.any([base, gate.signal]),
settle: () => clearTimeout(timer),
};
}
export interface RequestOpts {
/** Override the deadline. Pass one when the call is slow BY DESIGN. */
timeoutMs?: number;
}
/**
* The one place a request is actually made — every verb below goes through
* here, so the deadline cannot be forgotten by adding a sixth.
*
* EXPIRY SURFACES AS AN `ApiError`, the deadline's second half: the
* failure has to arrive in the shape the caller already handles. A bare
* `DOMException: TimeoutError` would reach `apiErrorMessage(e, fallback)` as
* an object with no `body`, so every catch site in the app would report its
* generic fallback and the timeout would be invisible in the very situation it
* exists to expose. Rethrowing as `ApiError` means ~330 existing call sites
* report it correctly without being touched.
*
* Only a TIMEOUT is converted. A deliberate cancellation aborts with
* `AbortError` and is left alone — a caller that cancelled its own request
* does not want it reported as a server failure.
*/
async function request<T>(path: string, init: RequestInit, opts?: RequestOpts): Promise<T> {
const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
let res: Response;
try {
res = await fetch(path, { ...init, signal: AbortSignal.timeout(timeoutMs) });
} catch (e) {
if (e instanceof DOMException && e.name === "TimeoutError") {
throw new ApiError(CLIENT_TIMEOUT_STATUS, {
error: `The server did not answer within ${Math.round(timeoutMs / 1000)}s.`,
});
}
throw e;
}
export async function apiGet<T>(path: string): Promise<T> {
const res = await fetch(path);
return handleResponse<T>(res, path);
}
/** JSON body headers — the three write verbs sent an identical literal each. */
const JSON_HEADERS = { "Content-Type": "application/json" };
export function apiGet<T>(path: string, opts?: RequestOpts): Promise<T> {
return request<T>(path, {}, opts);
export async function apiPost<T>(path: string, body: unknown): Promise<T> {
const res = await fetch(path, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
return handleResponse<T>(res, path);
}
export function apiPost<T>(path: string, body: unknown, opts?: RequestOpts): Promise<T> {
return request<T>(path, { method: "POST", headers: JSON_HEADERS, body: JSON.stringify(body) }, opts);
export async function apiPut<T>(path: string, body: unknown): Promise<T> {
const res = await fetch(path, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
return handleResponse<T>(res, path);
}
export function apiPut<T>(path: string, body: unknown, opts?: RequestOpts): Promise<T> {
return request<T>(path, { method: "PUT", headers: JSON_HEADERS, body: JSON.stringify(body) }, opts);
export async function apiPatch<T>(path: string, body: unknown): Promise<T> {
const res = await fetch(path, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
return handleResponse<T>(res, path);
}
export function apiPatch<T>(path: string, body: unknown, opts?: RequestOpts): Promise<T> {
return request<T>(path, { method: "PATCH", headers: JSON_HEADERS, body: JSON.stringify(body) }, opts);
}
export function apiDelete(path: string, opts?: RequestOpts): Promise<void> {
return request<void>(path, { method: "DELETE" }, opts);
export async function apiDelete(path: string): Promise<void> {
const res = await fetch(path, { method: "DELETE" });
return handleResponse<void>(res, path);
}
// ---------------------------------------------------------------------------
@@ -300,14 +207,7 @@ export function apiSSEStream(
}
const done = (async () => {
// Bounded connect, unbounded stream — see STREAM_CONNECT_TIMEOUT_MS.
const connect = connectDeadline(combinedSignal);
let res: Response;
try {
res = await fetch(path, { headers, signal: connect.signal });
} finally {
connect.settle();
}
const res = await fetch(path, { headers, signal: combinedSignal });
if (!res.ok) {
let body: Record<string, unknown> = {};
try {
@@ -404,19 +304,11 @@ export async function apiStreamPost(
body: unknown,
onChunk: (data: Record<string, unknown>) => void
): Promise<void> {
// Bounded connect, unbounded stream — see STREAM_CONNECT_TIMEOUT_MS.
const connect = connectDeadline(new AbortController().signal);
let res: Response;
try {
res = await fetch(path, {
method: "POST",
headers: JSON_HEADERS,
body: JSON.stringify(body),
signal: connect.signal,
});
} finally {
connect.settle();
}
const res = await fetch(path, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
if (!res.ok) {
let errBody: Record<string, unknown> = {};
try {
@@ -500,6 +392,24 @@ export const createApiKey = (name: string, scope: 'read' | 'write') =>
export const revokeApiKey = (id: number) => apiDelete(`/api/api-keys/${id}`)
// ── User Profile ─────────────────────────────────────────────────────────────
export interface UserProfile {
display_name: string
job_title: string
industry: string
expertise_level: 'novice' | 'intermediate' | 'expert'
response_style: 'concise' | 'balanced' | 'detailed'
tone: 'casual' | 'professional' | 'technical'
interests: string[]
work_schedule: { days?: string[]; start?: string; end?: string }
}
export const getProfile = () => apiGet<UserProfile>('/api/profile')
export const updateProfile = (data: Partial<UserProfile>) =>
apiPut<UserProfile>('/api/profile', data)
// ── Note Versions (pinning) ──────────────────────────────────────────────────
import type { NoteVersion } from '../types/task'
+9
View File
@@ -0,0 +1,9 @@
import { apiGet } from "@/api/client";
import type { ExpectationResponse } from "@/utils/designDrift";
/** Checkable claims from the rulebook this install designated as its design system.
*
* `rulebook_id: null` means none has been designated — the normal state for a
* fresh install, not an error. The caller shows an explanatory empty state. */
export const fetchDesignExpectations = () =>
apiGet<ExpectationResponse>("/api/design/expectations");
+4 -29
View File
@@ -1,5 +1,5 @@
/**
* Design systems — the stylesheet held as records.
* Design systems — the stylesheet held as records (milestone #254).
*
* A design system is a named set of tokens with an optional parent. A system
* with no parent is a "family"; one with a parent holds ONLY what it changes,
@@ -74,28 +74,11 @@ export const fetchDesignSystems = () =>
export const fetchDesignSystem = (id: number) =>
apiGet<DesignSystem>(`/api/design-systems/${id}`);
export interface StarterRoleGroup {
group: string;
description: string;
token_count: number;
names: string[];
}
/** The starter token ROLES offered at creation — names and purposes, never
* values. A default palette would be one install's taste shipped as product,
* so the values are always the operator's to fill. */
export const listStarterRoleGroups = () =>
apiGet<{ groups: StarterRoleGroup[]; default_prefix: string }>(
"/api/design-systems/starter-roles",
);
export const createDesignSystem = (body: {
title: string;
description?: string;
guidance?: string;
parent_id?: number | null;
starter_role_groups?: string[];
token_prefix?: string;
}) => apiPost<DesignSystem>("/api/design-systems", body);
/** Omit `parent_id` to leave it alone; send `null` to make the system a family. */
@@ -200,14 +183,6 @@ export interface SnippetCheck {
findings: SnippetFinding[];
}
/** Which recorded snippets disagree with this design system's sheet.
*
* `projectId` narrows to the snippets one project owns — which is how a
* project asks about its OWN code. Omit it to check every project, which is
* the right default from the system's side: a component recorded elsewhere
* still has to use the same tags. */
export const checkSnippets = (id: number, projectId?: number) =>
apiGet<SnippetCheck>(
`/api/design-systems/${id}/snippet-check`
+ (projectId ? `?project_id=${projectId}` : ""),
);
/** Which recorded snippets disagree with this design system's sheet. */
export const checkSnippets = (id: number) =>
apiGet<SnippetCheck>(`/api/design-systems/${id}/snippet-check`);
-169
View File
@@ -1,169 +0,0 @@
/**
* Family canon: ideas every project on a platform shares, and
* the log of every decision about them.
*
* The agent promotes by written criteria — no approval step. This door is for
* reading what was decided and why, and for retiring an idea or undoing a
* decision when a person disagrees.
*/
import { apiGet, apiPost } from "@/api/client";
export type IdeaStatus = "candidate" | "canon" | "retired";
export type DecisionAction = "propose" | "promote" | "revise" | "retire" | "assess" | "undo";
export interface Criterion {
key: string;
title: string;
test: string;
}
export interface IdeaTopic {
id: number;
title: string;
rulebook_id: number;
/** Present on a single idea's read, not on the list. */
rules?: { id: number; title: string }[];
}
export interface FamilyIdea {
note_id: number;
status: IdeaStatus;
applies_when: string;
canon_version: number;
topic_id: number | null;
/** The rule topic holding the idea's binding norms; null when none is
* linked or the reader does not own its rulebook. */
topic: IdeaTopic | null;
title: string;
note_type: string;
is_task: boolean;
project_id: number | null;
platforms: string[];
created_at: string | null;
updated_at: string | null;
}
/** A state snapshot a decision records — slugs, never ids. */
export interface IdeaSnapshot {
status: IdeaStatus;
applies_when: string;
canon_version: number;
platforms: string[];
}
export type AdoptionStatus = "unassessed" | "adopted" | "variant" | "exempt" | "owed";
/** One project's answer as a decision records it. */
export interface AdoptionSnapshot {
status: AdoptionStatus;
reason: string;
canon_version: number | null;
}
/**
* A decision about the idea itself (project_id null) snapshots the idea; a
* project's assessment (project_id set) snapshots that project's answer.
*/
export interface FamilyDecision {
id: number;
idea_id: number;
idea_title?: string;
project_id: number | null;
project_title?: string | null;
action: DecisionAction;
reason: string;
before: IdeaSnapshot | AdoptionSnapshot | null;
after: IdeaSnapshot | AdoptionSnapshot | null;
evidence: Record<string, unknown>;
precedent_ids: number[];
decided_via: "agent" | "operator" | "system";
user_id: number | null;
created_at: string | null;
undoable?: boolean;
}
/** One ground of the conflict order, in order. */
export interface ConflictGround {
key: string;
title: string;
test: string;
fold_as: "alternative" | "trap" | "condition";
}
export async function listFamilyIdeas(
status = "",
): Promise<{ ideas: FamilyIdea[]; criteria: Criterion[]; conflict_order?: ConflictGround[] }> {
const q = status ? `?status=${encodeURIComponent(status)}` : "";
return apiGet(`/api/family/ideas${q}`);
}
export async function listFamilyDecisions(limit = 50, offset = 0): Promise<FamilyDecision[]> {
const data = await apiGet<{ decisions: FamilyDecision[] }>(
`/api/family/decisions?limit=${limit}&offset=${offset}`,
);
return data.decisions;
}
export async function undoFamilyDecision(id: number, reason: string) {
return apiPost<{ decision: FamilyDecision }>(`/api/family/decisions/${id}/undo`, { reason });
}
export async function retireFamilyIdea(noteId: number, reason: string) {
return apiPost<{ decision: FamilyDecision }>(`/api/family/ideas/${noteId}/retire`, { reason });
}
export interface Outcome {
key: AdoptionStatus;
title: string;
test: string;
}
/** One project's answer to one canon idea — a cell of the matrix. */
export interface AdoptionCell {
project_id: number;
project_title: string;
idea_id: number;
idea_title: string;
idea_version: number;
/** The project is on one of the idea's platforms. */
in_scope: boolean;
/** A ledger row exists. False: the idea reaches the project, nobody asked yet. */
reached: boolean;
status: AdoptionStatus;
reason: string;
canon_version: number | null;
assessed_at: string | null;
decided_via: "agent" | "operator" | "system" | null;
/** Answered against a canon version that is not the current one. */
needs_recheck: boolean;
owed_task: { id: number; title: string; status: string } | null;
/** Live shapes in the project classified against one of the idea's references. */
shapes: { path: string; symbol: string; status: "canonical" | "instance" | "variant" }[];
}
export interface MatrixIdea {
note_id: number;
title: string;
note_type: string;
is_task: boolean;
canon_version: number;
applies_when: string;
platforms: string[];
}
export interface AdoptionMatrix {
ideas: MatrixIdea[];
projects: { id: number; title: string; platforms: string[] }[];
cells: AdoptionCell[];
outcomes: Outcome[];
}
export async function fetchAdoptionMatrix(
opts: { platform?: string; projectId?: number } = {},
): Promise<AdoptionMatrix> {
const q = new URLSearchParams();
if (opts.platform) q.set("platform", opts.platform);
if (opts.projectId) q.set("project_id", String(opts.projectId));
const qs = q.toString();
return apiGet(`/api/family/matrix${qs ? `?${qs}` : ""}`);
}

Some files were not shown because too many files have changed in this diff Show More