From 787685ac631ccc667c416e607391d770bd5d2def Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 22 Oct 2026 16:13:43 -0400 Subject: [PATCH] fix(6244): the instance keeps its own SECRET_KEY in /data, and docker-compose.yml is the compose users deploy from With no SECRET_KEY set, Grimoire fell back to a fixed placeholder. The quickstart compose substituted change-me-in-production, so the README's own update step (pull, up -d) recreated the container on that placeholder, and every credential sealed under the key the user started with stopped opening. - config: with neither SECRET_KEY nor SECRET_KEY_FILE, the first start writes a random key to DATA_DIR/secret_key (mode 0600, published with a hard link so concurrent starts agree), and later starts read it back. An explicit key wins and leaves the file alone. A missing SECRET_KEY_FILE or an empty key file stops the start instead of switching keys. With no data directory (a run outside the container) the process gets a temporary key and says so. - app: warns on a public placeholder key, on a temporary key, and when /data is not a mounted volume. - Dockerfile creates /data. - docker-compose.yml is now the deploy compose (published image, both volumes, nothing to set). The build-from-source compose is docker-compose.dev.yml, and docker-compose.quickstart.yml is gone. The Swarm stack file gains the /data volume and keeps its explicit key, because a Swarm can move the app away from a node-local volume. - README: a Deploy section with the compose inline, SECRET_KEY optional, "keep both volumes", and updates that keep the key. A test holds the README's copy equal to the file. - Installs that ran on the change-me-in-production placeholder get a key of their own on the new compose; Settings then asks once for saved forge tokens and the backup secret. Co-Authored-By: Claude Opus 5.5 --- .env.example | 4 +- Dockerfile | 4 + Makefile | 10 +-- README.md | 78 ++++++++++++----- ...e.quickstart.yml => docker-compose.dev.yml | 27 +++--- docker-compose.prod.yml | 6 ++ docker-compose.yml | 31 ++++--- docs/architecture.md | 2 +- docs/backups.md | 3 +- docs/configuration.md | 23 ++--- docs/development.md | 8 +- src/grimoire/app.py | 22 ++++- src/grimoire/config.py | 79 +++++++++++++++++- tests/test_config_secret_key.py | 83 +++++++++++++++++++ tests/test_readme_compose.py | 24 ++++++ 15 files changed, 329 insertions(+), 75 deletions(-) rename docker-compose.quickstart.yml => docker-compose.dev.yml (59%) create mode 100644 tests/test_config_secret_key.py create mode 100644 tests/test_readme_compose.py diff --git a/.env.example b/.env.example index 897c1f55..aa019006 100644 --- a/.env.example +++ b/.env.example @@ -4,4 +4,6 @@ POSTGRES_USER=fabled POSTGRES_PASSWORD=fabled POSTGRES_DB=grimoire # PGDATA_VOLUME=grimoire_pgdata -SECRET_KEY=dev-secret-change-me +# Grimoire keeps its own SECRET_KEY in the /data volume. Set one here only to +# use a key you manage yourself; it then wins over the kept one. +# SECRET_KEY= diff --git a/Dockerfile b/Dockerfile index d53ffedd..25777099 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,5 +56,9 @@ ENV APP_BUILD_KEY=$BUILD_KEY ENV APP_CHANNEL=$BUILD_CHANNEL ENV APP_COMMIT=$BUILD_COMMIT +# The instance's own SECRET_KEY, the model cache and diagnostics live here. +# Mount a volume on it so they outlive the container. +RUN mkdir -p /data + EXPOSE 5000 CMD ["sh", "-c", "alembic upgrade head && hypercorn 'grimoire.app:create_app()' --bind 0.0.0.0:5000 --keep-alive 600"] diff --git a/Makefile b/Makefile index cb7476ef..cdc7f970 100644 --- a/Makefile +++ b/Makefile @@ -3,22 +3,22 @@ # --- Docker --- build: - docker compose build + docker compose -f docker-compose.dev.yml build up: - docker compose up -d + docker compose -f docker-compose.dev.yml up -d down: - docker compose down + docker compose -f docker-compose.dev.yml down logs: - docker compose logs -f + docker compose -f docker-compose.dev.yml logs -f health: curl -s http://localhost:5000/api/health migrate: - docker compose exec app alembic upgrade head + docker compose -f docker-compose.dev.yml exec app alembic upgrade head # --- Local dev (requires: uv sync --extra dev and cd frontend && npm ci) --- diff --git a/README.md b/README.md index 8f2ad796..388d47d8 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,57 @@ Claude records what it learned, decided and built in Grimoire, and that record reaches the next session, so a solution is recalled instead of reinvented. Notes, tasks, plans, rules, lessons and reusable code live in one place. Claude reaches them through a built-in MCP endpoint and the bundled Claude Code plugin, and people read and edit the same records in the web UI. There is no language model inside Grimoire: Claude is the only assistant. -## Quick start +## Deploy -You need Docker with Compose. Download [`docker-compose.quickstart.yml`](docker-compose.quickstart.yml), then: +You need Docker with Compose. Save this as `docker-compose.yml` (it is the [`docker-compose.yml`](docker-compose.yml) in this repository), then run `docker compose up -d` beside it: -```bash -SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(32))') \ - docker compose -f docker-compose.quickstart.yml up -d +```yaml +# Fabled Grimoire: the app and its database. +# docker compose up -d +# Then open http://localhost:5000. The first account registered becomes admin. +# +# Keep both volumes. app_data holds the instance's own SECRET_KEY, which seals +# saved credentials; pgdata holds everything else. + +services: + app: + image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest + ports: + - "5000:5000" + environment: + DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}" + LOG_LEVEL: "${LOG_LEVEL:-INFO}" + # Behind an HTTPS reverse proxy: + # BASE_URL: "https://grimoire.example.com" + # SECURE_COOKIES: "true" + # TRUST_PROXY_HEADERS: "true" + volumes: + - app_data:/data + depends_on: + - db + restart: unless-stopped + + db: + # Postgres 17 with the vector extension Grimoire needs. + image: pgvector/pgvector:pg17 + stop_grace_period: 120s + volumes: + - pgdata:/var/lib/postgresql/data + environment: + POSTGRES_USER: ${POSTGRES_USER:-grimoire} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire} + POSTGRES_DB: ${POSTGRES_DB:-grimoire} + restart: unless-stopped + +volumes: + app_data: + pgdata: ``` Open `http://localhost:5000`. The first account you register becomes the admin. +Nothing has to be set first. On its first start Grimoire makes its own `SECRET_KEY` and keeps it in the `app_data` volume, so restarts, reboots and updates all use the same key. + To connect Claude, create an API key under **Settings → MCP Access**, then in Claude Code: ``` @@ -30,34 +70,34 @@ The plugin asks for your Grimoire URL and the API key. Settings → MCP Access s ## Configuration -Set these in the environment or a `.env` file beside the compose file: +Everything is optional. Set these in the environment or in a `.env` file beside the compose file: -| Variable | Needed | What it does | -|----------|--------|--------------| -| `SECRET_KEY` | Yes | Signs sessions and seals stored credentials. Set it once and keep it | +| Variable | When | What it does | +|----------|------|--------------| | `BASE_URL` | Behind a proxy | The public URL, used in email links and SSO redirects | | `SECURE_COOKIES` | Over HTTPS | `true` when Grimoire is served over HTTPS | | `TRUST_PROXY_HEADERS` | Behind a proxy | `true` to read the client address from the proxy's headers | -| `POSTGRES_PASSWORD` | Recommended | The database password (defaults to `grimoire`) | +| `POSTGRES_PASSWORD` | Before the first start | The database password (defaults to `grimoire`; the database is reachable only inside the compose network). Postgres takes it once, when the volume is created | +| `SECRET_KEY` | Rarely | A key you manage yourself, instead of the one Grimoire keeps. It wins over the kept key; remove it to go back | Everything else (registration, email, SSO, git forge connections, retrieval tuning, backups) is in **Settings**, or listed in [Configuration](docs/configuration.md). ## Setup pitfalls -- **Keep your `SECRET_KEY`.** Forge tokens and the backup bucket's secret are sealed under it. If it changes, Settings asks you to enter those credentials again. -- **The embedding model downloads on first use**, into the `/data` volume. Without internet access, semantic search falls back to keyword search until the model is in place. +- **Keep both volumes.** `pgdata` holds everything Grimoire stores. `app_data` holds the instance's key, and forge tokens and the backup bucket's secret are sealed under it. If the key is lost, Settings asks you to enter those credentials again; nothing else is lost. +- **The embedding model downloads on first use**, into the `app_data` volume. Without internet access, semantic search falls back to keyword search until the model is in place. - **Behind a reverse proxy, turn off response buffering** for `/mcp`, because MCP responses stream. [Configuration](docs/configuration.md#behind-a-reverse-proxy) has an Nginx example. ## Updating and backups ```bash -docker compose -f docker-compose.quickstart.yml pull -docker compose -f docker-compose.quickstart.yml up -d +docker compose pull +docker compose up -d ``` -Database migrations run when the app starts. Claude Code updates the plugin from its marketplace; run `/reload-plugins` after an update. +Database migrations run when the app starts, and the key stays in its volume. Claude Code updates the plugin from its marketplace; run `/reload-plugins` after an update. -Everything Grimoire stores is in the database. **Settings → Data → Scheduled backups** writes a daily backup to a folder or an S3-compatible bucket, and `python -m grimoire.restore` brings a fresh instance back from one. See [Backups](docs/backups.md). +**Settings → Data → Scheduled backups** writes a daily backup of the database to a folder or an S3-compatible bucket, and `python -m grimoire.restore` brings a fresh instance back from one. See [Backups](docs/backups.md). ## Features @@ -92,7 +132,8 @@ The full list is in [Features](docs/features.md). | What you see | Why | Fix | |--------------|-----|-----| -| `SECRET_KEY is set to the insecure default` at startup | `SECURE_COOKIES=true` while `SECRET_KEY` is still the built-in default | Set `SECRET_KEY` | +| `SECRET_KEY is set to the public placeholder` at startup | `SECURE_COOKIES=true` while `SECRET_KEY` is `dev-secret-change-me` | Remove `SECRET_KEY`, or set a real one | +| Settings asks for forge tokens or the backup secret again | The instance's key changed: the `app_data` volume was replaced, or `SECRET_KEY` was set or removed | Enter them again, or restore the previous key | | `Can't locate revision identified by '…'` at startup | An older image is running against a database a newer one has already migrated | Run the newer image; migrations only move forward | | The plugin's tools are missing in Claude Code | The plugin isn't installed, or Claude Code hasn't reloaded it | `/plugin` to install or update it, then `/reload-plugins` | | Search finds only exact words | The embedding model hasn't downloaded yet | Give the app internet access once, or wait for the first download to finish | @@ -100,8 +141,7 @@ The full list is in [Features](docs/features.md). ## Development ```bash -cp .env.example .env # then set SECRET_KEY -docker compose up --build +docker compose -f docker-compose.dev.yml up --build ``` [Development](docs/development.md) covers checks, migrations, the plugin and CI. [Architecture](docs/architecture.md) describes how it is built, and the [API reference](docs/api-reference.md) lists every REST endpoint. diff --git a/docker-compose.quickstart.yml b/docker-compose.dev.yml similarity index 59% rename from docker-compose.quickstart.yml rename to docker-compose.dev.yml index 90af1f63..e7fc02e4 100644 --- a/docker-compose.quickstart.yml +++ b/docker-compose.dev.yml @@ -1,36 +1,35 @@ -# Fabled Grimoire from the pre-built image: -# docker compose -f docker-compose.quickstart.yml up -d -# Then open http://localhost:5000 (the first account registered becomes admin) -# and connect Claude from Settings → MCP Access. +# Grimoire built from this checkout, for development: +# docker compose -f docker-compose.dev.yml up --build services: app: - image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest + build: . ports: - "5000:5000" - environment: - DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}" - # Set SECRET_KEY in the environment or a .env file beside this one. - SECRET_KEY: "${SECRET_KEY:-change-me-in-production}" - LOG_LEVEL: "${LOG_LEVEL:-INFO}" - volumes: - - app_data:/data depends_on: - db restart: unless-stopped + volumes: + - app_data:/data + # Or a bind mount, for direct host access to the app's data: + # - ./data:/data + environment: + DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}" + # Optional: a SearXNG instance to list as the web-search backend in Integrations. + # SEARXNG_URL: "http://searxng:8080" db: # Postgres 17 with the vector extension Grimoire needs. image: pgvector/pgvector:pg17 stop_grace_period: 120s + restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: POSTGRES_USER: ${POSTGRES_USER:-grimoire} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire} POSTGRES_DB: ${POSTGRES_DB:-grimoire} - restart: unless-stopped volumes: - app_data: pgdata: + app_data: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1d19d84c..a575577b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -3,10 +3,15 @@ services: image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest environment: DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${DB_PASSWORD}@db:5432/${POSTGRES_DB:-grimoire}" + # A Swarm can start the app on another node, away from the key kept in + # a node-local /data volume, so a stack sets its key itself (or mounts + # a Docker secret and sets SECRET_KEY_FILE to its path). SECRET_KEY: "${SECRET_KEY}" LOG_LEVEL: "${LOG_LEVEL:-INFO}" TRUST_PROXY_HEADERS: "true" SECURE_COOKIES: "true" + volumes: + - app_data:/data networks: - backend @@ -26,6 +31,7 @@ services: volumes: pgdata: name: ${PGDATA_VOLUME:-grimoire_pgdata} + app_data: networks: backend: diff --git a/docker-compose.yml b/docker-compose.yml index 2a420706..0a231d78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,33 +1,40 @@ +# Fabled Grimoire: the app and its database. +# docker compose up -d +# Then open http://localhost:5000. The first account registered becomes admin. +# +# Keep both volumes. app_data holds the instance's own SECRET_KEY, which seals +# saved credentials; pgdata holds everything else. + services: app: - build: . + image: git.fabledsword.com/bvandeusen/fabledgrimoire:latest ports: - "5000:5000" + environment: + DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}" + LOG_LEVEL: "${LOG_LEVEL:-INFO}" + # Behind an HTTPS reverse proxy: + # BASE_URL: "https://grimoire.example.com" + # SECURE_COOKIES: "true" + # TRUST_PROXY_HEADERS: "true" + volumes: + - app_data:/data depends_on: - db restart: unless-stopped - volumes: - - app_data:/data - # Or a bind mount, for direct host access to the app's data: - # - ./data:/data - environment: - DATABASE_URL: "postgresql+asyncpg://${POSTGRES_USER:-grimoire}:${POSTGRES_PASSWORD:-grimoire}@db:5432/${POSTGRES_DB:-grimoire}" - SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}" - # Optional: a SearXNG instance to list as the web-search backend in Integrations. - # SEARXNG_URL: "http://searxng:8080" db: # Postgres 17 with the vector extension Grimoire needs. image: pgvector/pgvector:pg17 stop_grace_period: 120s - restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: POSTGRES_USER: ${POSTGRES_USER:-grimoire} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-grimoire} POSTGRES_DB: ${POSTGRES_DB:-grimoire} + restart: unless-stopped volumes: - pgdata: app_data: + pgdata: diff --git a/docs/architecture.md b/docs/architecture.md index 735821cb..2ecc9e19 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,7 @@ Every database wait on startup has a deadline. Migrations wait for the database - **OIDC** single sign-on with PKCE, optionally replacing local logins ([SSO / OAuth](sso-oauth.md)). - **Rate limits** on register, login and forgot-password, per client IP (`TRUST_PROXY_HEADERS` reads the IP from the proxy's headers). -Credentials Grimoire stores for outside services (forge tokens, the backup destination's S3 secret) are sealed under a key derived from `SECRET_KEY`. +Credentials Grimoire stores for outside services (forge tokens, the backup destination's S3 secret) are sealed under a key derived from `SECRET_KEY`, which the instance makes on first start and keeps in `/data/secret_key` unless one is set. ## Repository layout diff --git a/docs/backups.md b/docs/backups.md index df93be32..e0c630cb 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -31,7 +31,8 @@ differently. It does **not** hold: - **Credentials:** API keys and forge connections. After a restore, issue new API keys and add forge connections again. (Forge tokens are also sealed under `SECRET_KEY` in the database itself, so a copy of the database doesn't carry - them in readable form either.) + them in readable form either. The key lives in `/data/secret_key`, outside + the database.) - **Logs and notifications.** - **The backup destination's own S3 secret.** diff --git a/docs/configuration.md b/docs/configuration.md index ac7e37ad..5c437bae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,8 @@ Any variable marked *secret* can also be read from a file: set `_FILE` to | Variable | Default | Description | |----------|---------|-------------| | `DATABASE_URL` *(secret)* | `postgresql+asyncpg://grimoire:grimoire@localhost:5432/grimoire` | PostgreSQL connection string, using the `asyncpg` driver | -| `SECRET_KEY` *(secret)* | `dev-secret-change-me` | Signs sessions and seals stored credentials. Set it before first use and keep it (see below) | +| `SECRET_KEY` *(secret)* | kept in `DATA_DIR` | Signs sessions and seals stored credentials. Unset, the instance makes its own on first start and keeps it in `DATA_DIR/secret_key`. Set it only to manage the key yourself; it wins over the kept one (see below) | +| `DATA_DIR` | `/data` | Where the instance keeps its own `SECRET_KEY`. Mount a volume here | | `BASE_URL` | `http://localhost:5000` | Public URL, used in email links and OIDC redirects | | `SECURE_COOKIES` | `false` | `true` when Grimoire is served over HTTPS | | `TRUST_PROXY_HEADERS` | `false` | `true` behind a reverse proxy, to read the client IP from `X-Forwarded-For` / `X-Real-IP` | @@ -56,27 +57,27 @@ Admins can set SMTP in **Settings**; these are the defaults until they do. ## Running it -### Quick start +### Docker Compose -[`docker-compose.quickstart.yml`](../docker-compose.quickstart.yml) runs the published image with its own database: +[`docker-compose.yml`](../docker-compose.yml) runs the published image with its own database, and needs nothing set: ```bash -SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(32))') \ - docker compose -f docker-compose.quickstart.yml up -d +docker compose up -d ``` Open `http://localhost:5000`. The first account registered becomes the admin; after that, registration stays closed until an admin opens it in Settings or sends an invite. +Update with `docker compose pull && docker compose up -d`. The key and the data stay in their volumes. + ### From source ```bash -cp .env.example .env # then set SECRET_KEY -docker compose up --build +docker compose -f docker-compose.dev.yml up --build ``` ### Docker Swarm -[`docker-compose.prod.yml`](../docker-compose.prod.yml) is a stack file: the app and database on a private overlay network, with `SECURE_COOKIES` and `TRUST_PROXY_HEADERS` on. Provide `DB_PASSWORD` and `SECRET_KEY` in the environment: +[`docker-compose.prod.yml`](../docker-compose.prod.yml) is a stack file: the app and database on a private overlay network, with `SECURE_COOKIES` and `TRUST_PROXY_HEADERS` on. A Swarm can move the app to another node, away from a node-local `/data` volume, so the stack sets its own key: provide `DB_PASSWORD` and `SECRET_KEY` in the environment (or mount a Docker secret and set `SECRET_KEY_FILE`): ```bash DB_PASSWORD=... SECRET_KEY=... docker stack deploy -c docker-compose.prod.yml grimoire @@ -89,9 +90,9 @@ The shipped files leave out healthchecks and restart policies on purpose. The ap ### Volumes - **Database** (`pgdata`): everything Grimoire stores. -- **App data** (`/data`): the embedding model cache and diagnostics. Losing it costs a re-download, nothing else. +- **App data** (`/data`): the instance's own `SECRET_KEY`, the embedding model cache and diagnostics. Losing it costs the key: Settings then asks for saved forge tokens and the backup secret again, and the model downloads again. -Back up the database with **Settings → Data → Scheduled backups** ([Backups](backups.md)). +Back up the database with **Settings → Data → Scheduled backups** ([Backups](backups.md)). A database backup doesn't carry the key; to move sealed credentials to a new instance, copy `/data/secret_key` with it. ## Behind a reverse proxy @@ -115,7 +116,7 @@ With `TRUST_PROXY_HEADERS=true`, make sure the proxy replaces any `X-Forwarded-F ## Security checklist -- **Set a strong `SECRET_KEY`, and keep it.** Forge tokens and the backup destination's S3 secret are sealed under a key derived from it, so a copy of the database alone can't read them. If the key changes (say, a restore onto an instance with a different key), Settings marks those credentials and asks for them again; nothing else breaks. The app logs a warning at startup while the default key is in use, and refuses to start with it when `SECURE_COOKIES=true`. +- **Keep the key.** Forge tokens and the backup destination's S3 secret are sealed under a key derived from `SECRET_KEY`, so a copy of the database alone can't read them. Unset, the instance makes a random key and keeps it in `/data/secret_key`, readable by its owner only; keep that volume. If the key changes (say, a restore onto an instance with a different key), Settings marks those credentials and asks for them again; nothing else breaks. The app warns at startup when `SECRET_KEY` is a public placeholder or `/data` is not a mounted volume, and refuses to start with `dev-secret-change-me` when `SECURE_COOKIES=true`. - **`SECURE_COOKIES=true`** whenever Grimoire is served over HTTPS. - **Registration** is open only for the first account. Invite people, or open it briefly from Settings. - **Signing everyone else out**: changing a password ends that user's other sessions; Settings → Account has a button for it too. diff --git a/docs/development.md b/docs/development.md index 096e4ebf..d0ec56a1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -3,9 +3,9 @@ ## Running it ```bash -docker compose up --build # app + PostgreSQL at http://localhost:5000 -docker compose up --build app # rebuild after a change -docker compose down -v # stop and wipe the database +docker compose -f docker-compose.dev.yml up --build # app + PostgreSQL at http://localhost:5000 +docker compose -f docker-compose.dev.yml up --build app # rebuild after a change +docker compose -f docker-compose.dev.yml down -v # stop and wipe the database ``` The image builds the frontend, so a frontend change needs a rebuild too. For quicker frontend work, run Vite against the running app; it proxies `/api` to port 5000: @@ -32,7 +32,7 @@ Unit tests run without a database. Tests marked `integration` need a real Postgr The container runs `alembic upgrade head` on every start, after waiting for the database to accept connections. ```bash -docker compose exec app alembic revision -m "short_description" +docker compose -f docker-compose.dev.yml exec app alembic revision -m "short_description" ``` - Number revisions sequentially (`0123_...`). Revision ids must fit in 32 characters, the width of `alembic_version.version_num`. diff --git a/src/grimoire/app.py b/src/grimoire/app.py index 21e36252..6d278ede 100644 --- a/src/grimoire/app.py +++ b/src/grimoire/app.py @@ -1,11 +1,12 @@ import logging +import os import time import traceback as tb_module from pathlib import Path from quart import Quart, g, jsonify, make_response, request, send_from_directory -from grimoire.config import Config +from grimoire.config import PLACEHOLDER_SECRET_KEYS, SECRET_KEY_FILENAME, Config from grimoire.routes.admin import admin_bp from grimoire.routes.api import api from grimoire.routes.auth import auth_bp @@ -62,10 +63,23 @@ def create_app() -> Quart: app.config["SESSION_COOKIE_SAMESITE"] = "Lax" app.config["SESSION_COOKIE_SECURE"] = Config.SECURE_COOKIES - if Config.SECRET_KEY == "dev-secret-change-me": + if Config.SECRET_KEY in PLACEHOLDER_SECRET_KEYS: logger.warning( - "SECRET_KEY is set to the default value — session cookies are insecure. " - "Set SECRET_KEY or SECRET_KEY_FILE for production use." + "SECRET_KEY is set to a public placeholder, so session cookies are insecure. " + "Remove it to let the instance keep its own key in %s, or set a real one.", + Config.DATA_DIR, + ) + if Config.SECRET_KEY_SOURCE == "ephemeral": + logger.warning( + "No SECRET_KEY is set and %s does not exist, so this process is using a " + "temporary key. Sessions and saved credentials will not survive a restart.", + Config.DATA_DIR, + ) + elif Config.SECRET_KEY_SOURCE == "data" and not os.path.ismount(Config.DATA_DIR): + logger.warning( + "%s is not a mounted volume. The instance's key (%s) and the model cache " + "are lost when the container is recreated; mount a volume there.", + Config.DATA_DIR, SECRET_KEY_FILENAME, ) if not Config.TRUST_PROXY_HEADERS: diff --git a/src/grimoire/config.py b/src/grimoire/config.py index 111351c4..d0a11a62 100644 --- a/src/grimoire/config.py +++ b/src/grimoire/config.py @@ -1,4 +1,13 @@ import os +import secrets +from pathlib import Path + +# The file in the data directory that holds the instance's own SECRET_KEY. +SECRET_KEY_FILENAME = "secret_key" + +# Placeholder keys that sample files and older compose files have shipped. A +# key anyone can read in this repository signs nothing safely. +PLACEHOLDER_SECRET_KEYS = frozenset({"dev-secret-change-me", "change-me-in-production"}) def _read_secret(env_var: str, secret_file_var: str, default: str) -> str: @@ -16,13 +25,76 @@ def _read_secret(env_var: str, secret_file_var: str, default: str) -> str: return default +def load_or_create_secret_key(data_dir: Path) -> str | None: + """The instance's own SECRET_KEY, kept in `data_dir` and made on first use. + + The first call writes a random key to `/secret_key`, readable by + its owner only; every later call, from this process or another, reads that + same key back. The file is published with a hard link, so two processes + starting together agree on one key and a crash never leaves an empty file. + Answers None when `data_dir` doesn't exist, which is a run outside the + container. An empty key file raises rather than being replaced, because a + new key would orphan everything sealed under the old one. + """ + if not data_dir.is_dir(): + return None + path = data_dir / SECRET_KEY_FILENAME + if not path.exists(): + tmp = data_dir / f".{SECRET_KEY_FILENAME}.{os.getpid()}" + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + with os.fdopen(fd, "w") as f: + f.write(secrets.token_hex(32) + "\n") + f.flush() + os.fsync(f.fileno()) + try: + os.link(tmp, path) + except FileExistsError: + pass + finally: + tmp.unlink(missing_ok=True) + key = path.read_text().strip() + if not key: + raise RuntimeError( + f"{path} is empty. Restore it from a copy, or delete it to start " + "with a new key (saved forge tokens and the backup secret are then " + "entered again)." + ) + return key + + +def _secret_key() -> tuple[str, str]: + """SECRET_KEY and where it came from: "env", "data" or "ephemeral". + + An explicit key (SECRET_KEY or SECRET_KEY_FILE) wins and leaves the data + directory's key file alone, so unsetting it returns to that key. With no + explicit key the instance keeps its own in DATA_DIR. With neither, the + process gets a random key of its own that dies with it. + """ + explicit = _read_secret("SECRET_KEY", "SECRET_KEY_FILE", "") + if explicit: + return explicit, "env" + if os.environ.get("SECRET_KEY_FILE"): + # Falling through to another key would orphan what the named one sealed. + raise RuntimeError( + f"SECRET_KEY_FILE={os.environ['SECRET_KEY_FILE']} is missing or empty." + ) + kept = load_or_create_secret_key(Path(os.environ.get("DATA_DIR", "/data"))) + if kept: + return kept, "data" + return secrets.token_hex(32), "ephemeral" + + class Config: DATABASE_URL: str = _read_secret( "DATABASE_URL", "DATABASE_URL_FILE", "postgresql+asyncpg://grimoire:grimoire@localhost:5432/grimoire", ) - SECRET_KEY: str = _read_secret("SECRET_KEY", "SECRET_KEY_FILE", "dev-secret-change-me") + # Where the instance keeps its own SECRET_KEY. /data is a volume in every + # shipped compose file, beside the model cache and diagnostics. + DATA_DIR: str = os.environ.get("DATA_DIR", "/data") + SECRET_KEY, SECRET_KEY_SOURCE = _secret_key() SECURE_COOKIES: bool = os.environ.get("SECURE_COOKIES", "").lower() in ("1", "true", "yes") LOG_LEVEL: str = os.environ.get("LOG_LEVEL", "INFO") LOG_RETENTION_DAYS: int = int(os.environ.get("LOG_RETENTION_DAYS", "90")) @@ -93,8 +165,9 @@ class Config: errors.append(f"BASE_URL='{cls.BASE_URL}' must start with http:// or https:// when OIDC is enabled") if cls.SECRET_KEY == "dev-secret-change-me" and cls.SECURE_COOKIES: errors.append( - "SECRET_KEY is set to the insecure default but SECURE_COOKIES=true indicates " - "a production deployment. Set SECRET_KEY or SECRET_KEY_FILE before starting." + "SECRET_KEY is set to the public placeholder 'dev-secret-change-me' but " + "SECURE_COOKIES=true indicates a production deployment. Remove SECRET_KEY " + "to let the instance keep its own key in DATA_DIR, or set a real one." ) if errors: raise ValueError("Configuration errors:\n" + "\n".join(f" - {e}" for e in errors)) diff --git a/tests/test_config_secret_key.py b/tests/test_config_secret_key.py new file mode 100644 index 00000000..f6c9eecf --- /dev/null +++ b/tests/test_config_secret_key.py @@ -0,0 +1,83 @@ +"""The instance keeps its own SECRET_KEY in the data directory. + +A deployment that never sets SECRET_KEY must get the same key on every start, +or everything sealed under the first one (forge tokens, the backup secret) +stops opening after the first container recreate. +""" +import stat +from unittest.mock import patch + +import pytest + +from grimoire import config +from grimoire.config import SECRET_KEY_FILENAME, load_or_create_secret_key +from grimoire.services import secrets + + +def test_the_first_start_writes_a_key_only_its_owner_can_read(tmp_path): + key = load_or_create_secret_key(tmp_path) + + path = tmp_path / SECRET_KEY_FILENAME + assert key and len(key) == 64 + assert path.read_text().strip() == key + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert [p.name for p in tmp_path.iterdir()] == [SECRET_KEY_FILENAME] + + +def test_every_later_start_reads_the_same_key_and_what_it_sealed_still_opens(tmp_path): + first = load_or_create_secret_key(tmp_path) + with patch.object(secrets.Config, "SECRET_KEY", first): + sealed = secrets.seal("forge-token") + + second = load_or_create_secret_key(tmp_path) + + assert second == first + with patch.object(secrets.Config, "SECRET_KEY", second): + assert secrets.unseal(sealed) == "forge-token" + + +def test_no_data_directory_means_no_kept_key(tmp_path): + assert load_or_create_secret_key(tmp_path / "absent") is None + + +def test_an_empty_key_file_is_refused_rather_than_replaced(tmp_path): + (tmp_path / SECRET_KEY_FILENAME).write_text("\n") + + with pytest.raises(RuntimeError, match="is empty"): + load_or_create_secret_key(tmp_path) + assert (tmp_path / SECRET_KEY_FILENAME).read_text() == "\n" + + +def test_an_explicit_key_wins_and_leaves_the_kept_one_alone(tmp_path, monkeypatch): + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.delenv("SECRET_KEY_FILE", raising=False) + monkeypatch.delenv("SECRET_KEY", raising=False) + kept, source = config._secret_key() + assert source == "data" + + monkeypatch.setenv("SECRET_KEY", "an-operator-key") + assert config._secret_key() == ("an-operator-key", "env") + + monkeypatch.delenv("SECRET_KEY") + assert config._secret_key() == (kept, "data") + + +def test_a_secret_key_file_that_is_missing_stops_the_start(tmp_path, monkeypatch): + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.delenv("SECRET_KEY", raising=False) + monkeypatch.setenv("SECRET_KEY_FILE", str(tmp_path / "no-such-secret")) + + with pytest.raises(RuntimeError, match="SECRET_KEY_FILE"): + config._secret_key() + assert not (tmp_path / SECRET_KEY_FILENAME).exists() + + +def test_with_nowhere_to_keep_a_key_the_process_gets_a_temporary_one(tmp_path, monkeypatch): + monkeypatch.setenv("DATA_DIR", str(tmp_path / "absent")) + monkeypatch.delenv("SECRET_KEY", raising=False) + monkeypatch.delenv("SECRET_KEY_FILE", raising=False) + + key, source = config._secret_key() + + assert source == "ephemeral" + assert key not in config.PLACEHOLDER_SECRET_KEYS and len(key) == 64 diff --git a/tests/test_readme_compose.py b/tests/test_readme_compose.py new file mode 100644 index 00000000..c2b7f3c7 --- /dev/null +++ b/tests/test_readme_compose.py @@ -0,0 +1,24 @@ +"""The compose file the README shows is the one users deploy from. + +The README carries docker-compose.yml inline, so a newcomer can copy it from +the page. Both copies have to say the same thing. +""" +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_the_readme_shows_the_shipped_compose_file_verbatim(): + compose = (ROOT / "docker-compose.yml").read_text() + readme = (ROOT / "README.md").read_text() + + assert f"```yaml\n{compose}```" in readme + + +def test_the_shipped_compose_needs_no_key_and_keeps_its_data(): + compose = (ROOT / "docker-compose.yml").read_text() + + settings = [line for line in compose.splitlines() if not line.lstrip().startswith("#")] + assert not [line for line in settings if "SECRET_KEY" in line] + assert "app_data:/data" in compose + assert "pgdata:/var/lib/postgresql/data" in compose