rename(547): the server is grimoire — package, image, DB defaults, backups

Step 2 of the Scribe -> Grimoire rename.

- src/scribe -> src/grimoire, and every import, patch target, module path,
  hypercorn entry point and `python -m grimoire.restore` with it; pyproject,
  uv.lock and the frontend package are named grimoire.
- Image git.fabledsword.com/bvandeusen/fabledgrimoire; CI's Postgres service
  runs as grimoire / grimoire_test.
- Compose files default the DB user, database and (prod) volume to grimoire,
  each overridable from .env, so an install created as Scribe keeps its data
  by setting POSTGRES_USER, POSTGRES_DB and PGDATA_VOLUME.
- Backups are written as grimoire-<ts>.tar with format grimoire-backup-tar.
  Archives named scribe-* with the scribe-backup-tar manifest still restore
  and are still counted at the destination (folder and S3), with tests.
- Exports, the MCP auth realm, the ASGI scope key and the persistent logger
  carry the new name.

Kept on purpose: the secret-sealing key label stays "scribe.secrets.v1",
because every stored secret was derived under it.

Plugin identity (plugin name, hooks, env vars, .scribe marker, skills) is
step 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/grimoire/app.py

@@ -0,0 +1,405 @@
+import logging
+import time
+import traceback as tb_module
+from pathlib import Path
+
+from quart import Quart, g, jsonify, make_response, request, send_from_directory
+
+from grimoire.config import Config
+from grimoire.routes.admin import admin_bp
+from grimoire.routes.api import api
+from grimoire.routes.auth import auth_bp
+from grimoire.routes.export import export_bp
+from grimoire.routes.notes import notes_bp
+from grimoire.routes.milestones import milestones_bp
+from grimoire.routes.task_logs import task_logs_bp
+from grimoire.routes.projects import projects_bp
+from grimoire.routes.retrieval import retrieval_bp
+from grimoire.routes.settings import settings_bp
+from grimoire.routes.tasks import tasks_bp
+from grimoire.routes.groups import groups_bp
+from grimoire.routes.shares import shares_bp
+from grimoire.routes.in_app_notifications import notifications_bp
+from grimoire.routes.users import users_bp
+from grimoire.routes.api_keys import api_keys_bp
+from grimoire.routes.search import search_bp
+from grimoire.routes.knowledge import knowledge_bp
+from grimoire.routes.rulebooks import rulebooks_bp
+from grimoire.routes.plugin import plugin_bp
+from grimoire.routes.design_systems import design_systems_bp
+from grimoire.routes.trash import trash_bp
+from grimoire.routes.dashboard import dashboard_bp
+from grimoire.routes.systems import systems_bp
+from grimoire.routes.canonical_systems import canonical_systems_bp
+from grimoire.routes.platforms import platforms_bp
+from grimoire.routes.family import family_bp
+from grimoire.routes.lessons import lessons_bp
+from grimoire.routes.snippets import snippets_bp
+from grimoire.routes.webhooks import webhooks_bp
+from grimoire.mcp import mount_mcp
+
+STATIC_DIR = Path(__file__).parent / "static"
+logger = logging.getLogger(__name__)
+
+
+def create_app() -> Quart:
+    # Configure logging
+    logging.basicConfig(
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        level=getattr(logging, Config.LOG_LEVEL.upper(), logging.INFO),
+    )
+
+    # Validate config early so misconfigurations surface immediately with clear messages
+    try:
+        Config.validate()
+    except ValueError as exc:
+        logging.getLogger(__name__).error("Invalid configuration:\n%s", exc)
+        raise
+
+    app = Quart(__name__, static_folder=None)
+    app.secret_key = Config.SECRET_KEY
+    app.config["SESSION_COOKIE_HTTPONLY"] = True
+    app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
+    app.config["SESSION_COOKIE_SECURE"] = Config.SECURE_COOKIES
+
+    if Config.SECRET_KEY == "dev-secret-change-me":
+        logger.warning(
+            "SECRET_KEY is set to the default value — session cookies are insecure. "
+            "Set SECRET_KEY or SECRET_KEY_FILE for production use."
+        )
+
+    if not Config.TRUST_PROXY_HEADERS:
+        logger.warning(
+            "TRUST_PROXY_HEADERS is not set. If this instance is behind a reverse proxy "
+            "(nginx, Caddy, Traefik) set TRUST_PROXY_HEADERS=true so rate limiting uses "
+            "real client IPs rather than the proxy IP."
+        )
+
+    app.register_blueprint(admin_bp)
+    app.register_blueprint(api)
+    app.register_blueprint(auth_bp)
+    app.register_blueprint(export_bp)
+    app.register_blueprint(milestones_bp)
+    app.register_blueprint(notes_bp)
+    app.register_blueprint(projects_bp)
+    app.register_blueprint(retrieval_bp)
+    app.register_blueprint(settings_bp)
+    app.register_blueprint(task_logs_bp)
+    app.register_blueprint(tasks_bp)
+    app.register_blueprint(groups_bp)
+    app.register_blueprint(shares_bp)
+    app.register_blueprint(notifications_bp)
+    app.register_blueprint(users_bp)
+    app.register_blueprint(api_keys_bp)
+    app.register_blueprint(search_bp)
+    app.register_blueprint(knowledge_bp)
+    app.register_blueprint(lessons_bp)
+    app.register_blueprint(rulebooks_bp)
+    app.register_blueprint(plugin_bp)
+    app.register_blueprint(design_systems_bp)
+    app.register_blueprint(trash_bp)
+    app.register_blueprint(dashboard_bp)
+    app.register_blueprint(systems_bp)
+    app.register_blueprint(canonical_systems_bp)
+    app.register_blueprint(platforms_bp)
+    app.register_blueprint(family_bp)
+    app.register_blueprint(snippets_bp)
+    app.register_blueprint(webhooks_bp)
+
+    @app.before_request
+    async def before_request():
+        g.request_start = time.monotonic()
+
+    @app.after_request
+    async def after_request(response):
+        duration = time.monotonic() - getattr(g, "request_start", time.monotonic())
+        duration_ms = round(duration * 1000, 1)
+        # Downgrade noisy high-frequency / static paths to DEBUG
+        _quiet = (
+            request.path in {"/api/health", "/api/chat/status"}
+            or request.path.startswith(("/static/", "/assets/", "/sw.js", "/manifest.json"))
+        )
+        log_fn = logger.debug if _quiet else logger.info
+        log_fn(
+            "%s %s %s %.1fms",
+            request.method,
+            request.path,
+            response.status_code,
+            duration_ms,
+        )
+
+        # Log usage for API requests (skip logs endpoint to avoid recursion)
+        if request.path.startswith("/api/") and not request.path.startswith("/api/admin/logs"):
+            try:
+                from grimoire.services.logging import log_usage
+
+                user = getattr(g, "user", None)
+                await log_usage(
+                    user_id=user.id if user else None,
+                    username=user.username if user else None,
+                    endpoint=request.path,
+                    method=request.method,
+                    status_code=response.status_code,
+                    duration_ms=duration_ms,
+                )
+            except Exception:
+                logger.debug("Failed to log usage", exc_info=True)
+
+        response.headers.setdefault("X-Content-Type-Options", "nosniff")
+        response.headers.setdefault("X-Frame-Options", "DENY")
+        response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
+        response.headers.setdefault(
+            "Content-Security-Policy",
+            "default-src 'self'; "
+            "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval'; "
+            "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
+            "img-src 'self' data: blob:; "
+            "connect-src 'self'; "
+            "font-src 'self' data: https://fonts.gstatic.com; "
+            "object-src 'none'; "
+            "base-uri 'self'; worker-src 'self' blob:;"
+        )
+
+        return response
+
+    @app.before_serving
+    async def startup():
+        import asyncio
+
+        # Set on the last line of this hook; the deferred backfill below waits
+        # on it. An Event rather than a bare bool so the waiter is woken
+        # instead of polling, and declared here — inside the hook, where
+        # `asyncio` is in scope and a running loop exists — rather than in
+        # `create_app`, which runs before either is true.
+        _startup_finished = asyncio.Event()
+
+        from grimoire.services.auth import start_auth_token_retention_loop
+        from grimoire.services.embeddings import (
+            backfill_milestone_embeddings, backfill_note_embeddings,
+            backfill_rule_embeddings, backfill_system_embeddings,
+        )
+        from grimoire.services.logging import start_log_retention_loop
+        from grimoire.services.notifications import start_notification_loop
+
+        start_log_retention_loop()
+        start_notification_loop()
+        start_auth_token_retention_loop()
+
+        # Write down any shipped default that moved in this release.
+        #
+        # INLINE AND AWAITED, deliberately, against the instinct the block
+        # below argues for. The startup hazard is CONCURRENCY — a background
+        # task created here racing this hook for the same pool. Running
+        # sequentially creates no such contention, and this is a dozen
+        # single-row reads on an index against a table with a handful of rows.
+        #
+        # It has to finish before serving because the thing it protects is a
+        # telemetry read: `band_hugs_floor` compares a band against a floor,
+        # and across a release that changed the floor those describe different
+        # regimes. A readout served before the change was recorded is the very
+        # answer this exists to stop giving. Failure is logged and swallowed —
+        # provenance is worth a lot and never worth refusing to boot.
+        try:
+            from grimoire.services.retrieval_tuning import record_release_defaults
+            moved = await record_release_defaults()
+            for row in moved:
+                if not row["baseline"]:
+                    app.logger.info(
+                        "release moved %s's shipped %s: %s -> %s",
+                        row["surface"], row["dial"],
+                        row["old_value"], row["new_value"],
+                    )
+        except Exception:
+            app.logger.warning("could not record release defaults", exc_info=True)
+
+        # Backfill embeddings for any notes that don't have one. Runs in the
+        # background so it never blocks the server from accepting requests —
+        # and not until the rest of this hook has finished.
+        #
+        # "Background" is true of REQUESTS and false of STARTUP. A task
+        # created here begins immediately, while `before_serving` is still
+        # running, so its passes would compete with the startup hook's own
+        # database reads for the same connection pool. That is survivable on a
+        # healthy disk and fatal on a sick one: Hypercorn kills a worker whose
+        # lifespan hook overruns, and nothing retries it.
+        #
+        # The wait is on the SERVING FLAG rather than a sleep, because a sleep
+        # would be a guess about how long the rest of the hook takes and would
+        # be wrong in exactly the conditions that matter.
+        async def _delayed_backfill() -> None:
+            await _startup_finished.wait()
+            try:
+                await backfill_note_embeddings()
+            except Exception:
+                logger.warning("Embedding backfill failed", exc_info=True)
+            # A rule with no vector is unfindable by meaning; this pass fills
+            # any that are missing.
+            try:
+                await backfill_rule_embeddings()
+            except Exception:
+                logger.warning("Rule embedding backfill failed", exc_info=True)
+            # Likewise a milestone: a plan with no vector is findable only
+            # after this pass.
+            try:
+                await backfill_milestone_embeddings()
+            except Exception:
+                logger.warning("Milestone embedding backfill failed", exc_info=True)
+            # A System charter with no vector is unfindable — the System could
+            # narrow a search and never be the answer to one.
+            try:
+                await backfill_system_embeddings()
+            except Exception:
+                logger.warning("System embedding backfill failed", exc_info=True)
+            # Snippets written before migration 0070 have no `notes.data` mirror,
+            # and the location reverse lookup queries that column — an unfilled
+            # row would read as "no snippet here" rather than as a gap. Separate
+            # try block so neither backfill can skip the other.
+            try:
+                from grimoire.services.snippets import backfill_snippet_data
+                await backfill_snippet_data()
+            except Exception:
+                logger.warning("Snippet data backfill failed", exc_info=True)
+
+        # Created here but gated on the flag released at the END of this hook,
+        # so the task exists (nothing can forget to start it) while none of its
+        # work overlaps startup's own.
+        asyncio.create_task(_delayed_backfill())
+
+        # RELEASED IN A `finally`, never after the work.
+        # The waiter above has no deadline of its own, and the way to make an
+        # undeadlined wait safe is to make the release unmissable: if anything
+        # below raises, the flag is still set and the task ends instead of
+        # living on as a coroutine nobody will ever wake.
+        try:
+            # Recurrence scheduler (recurring-task spawn every 15m)
+            from grimoire.services.recurrence_scheduler import start_recurrence_scheduler
+            start_recurrence_scheduler(asyncio.get_running_loop())
+
+            # Version-pinning scheduler (daily auto-pin scan at 03:00 UTC)
+            from grimoire.services.version_pinning_scheduler import (
+                start_version_pinning_scheduler,
+            )
+            start_version_pinning_scheduler(asyncio.get_running_loop())
+
+            # Trash retention scheduler (daily expired-trash purge at 03:30 UTC)
+            from grimoire.services.trash_scheduler import start_trash_scheduler
+            start_trash_scheduler(asyncio.get_running_loop())
+
+            # DB maintenance scheduler (daily targeted VACUUM ANALYZE, default
+            # 04:00 UTC). `get_maintenance_hour` is the FIRST database read in
+            # this hook and is bounded for that reason — see the long
+            # comment above `_STARTUP_READ_TIMEOUT`. Anything added here that
+            # touches the database needs the same treatment: a lifespan hook
+            # that does not return is a worker that never serves.
+            from grimoire.services.db_maintenance_scheduler import (
+                get_maintenance_hour,
+                start_db_maintenance_scheduler,
+            )
+            start_db_maintenance_scheduler(
+                asyncio.get_running_loop(), await get_maintenance_hour()
+            )
+
+            # Scheduled backups. An hourly tick that reads its config when it
+            # fires, so starting it reads nothing here.
+            from grimoire.services.backup_job import start_backup_scheduler
+            start_backup_scheduler(asyncio.get_running_loop())
+
+            # Diagnostic instrumentation — heartbeat, signal handlers, asyncio
+            # exception hook. Cheap (~1 log line/min), high diagnostic value when
+            # the app crashes mysteriously. See services/diagnostics.py.
+            from grimoire.services.diagnostics import start_diagnostics
+            start_diagnostics(asyncio.get_running_loop())
+        finally:
+            # STARTUP IS OVER — release the backfill. Everything above
+            # is work the server needs done before it serves; everything the
+            # backfill does is work that can wait for a server already up.
+            logger.info("Startup complete; releasing deferred backfill")
+            _startup_finished.set()
+
+    @app.after_serving
+    async def shutdown():
+        from grimoire.services.recurrence_scheduler import stop_recurrence_scheduler
+        stop_recurrence_scheduler()
+        from grimoire.services.version_pinning_scheduler import (
+            stop_version_pinning_scheduler,
+        )
+        stop_version_pinning_scheduler()
+        from grimoire.services.trash_scheduler import stop_trash_scheduler
+        stop_trash_scheduler()
+        from grimoire.services.db_maintenance_scheduler import (
+            stop_db_maintenance_scheduler,
+        )
+        stop_db_maintenance_scheduler()
+        from grimoire.services.backup_job import stop_backup_scheduler
+        stop_backup_scheduler()
+        from grimoire.services.diagnostics import stop_diagnostics
+        stop_diagnostics()
+
+    @app.route("/")
+    async def serve_index():
+        resp = await make_response(
+            await send_from_directory(STATIC_DIR, "index.html")
+        )
+        resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+        return resp
+
+    # File extensions that belong to the SPA or its assets.
+    # Anything else with an extension is not a valid app path and gets a hard 404.
+    _SPA_EXTENSIONS = {
+        "", ".html", ".js", ".css", ".ico", ".png", ".jpg", ".jpeg",
+        ".svg", ".webp", ".woff", ".woff2", ".ttf", ".otf", ".map", ".json",
+    }
+
+    @app.errorhandler(404)
+    async def handle_404(error):
+        # Return JSON 404 for API routes
+        if request.path.startswith("/api/"):
+            return jsonify({"error": "Not found"}), 404
+
+        # Try to serve a real static file first
+        path = request.path.lstrip("/")
+        file_path = STATIC_DIR / path
+        if path and file_path.is_file():
+            return await send_from_directory(STATIC_DIR, path)
+
+        # Reject paths with file extensions the app doesn't serve.
+        # This turns scanner probes (.php, .asp, .cgi, etc.) into honest 404s
+        # instead of serving them the Vue SPA with a misleading 200.
+        from pathlib import PurePosixPath
+        suffix = PurePosixPath(request.path).suffix.lower()
+        if suffix not in _SPA_EXTENSIONS:
+            return "", 404
+
+        # SPA fallback for clean client-side routes (/notes/123, /chat, etc.)
+        resp = await make_response(
+            await send_from_directory(STATIC_DIR, "index.html")
+        )
+        resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+        return resp
+
+    @app.errorhandler(500)
+    async def handle_500(error):
+        logger.exception("Internal server error on %s %s", request.method, request.path)
+
+        try:
+            from grimoire.services.logging import log_error
+
+            user = getattr(g, "user", None)
+            await log_error(
+                user_id=user.id if user else None,
+                username=user.username if user else None,
+                endpoint=request.path,
+                method=request.method,
+                error_type=type(error).__name__,
+                error_message=str(error),
+                traceback=tb_module.format_exc(),
+            )
+        except Exception:
+            logger.debug("Failed to log error", exc_info=True)
+
+        if request.path.startswith("/api/"):
+            return jsonify({"error": "Internal server error"}), 500
+        return "Internal Server Error", 500
+
+    mount_mcp(app)
+    return app