FabledScribe/src/scribe/app.py

import logging
import time
import traceback as tb_module
from pathlib import Path

from quart import Quart, g, jsonify, make_response, request, send_from_directory

from scribe.config import Config
from scribe.routes.admin import admin_bp
from scribe.routes.api import api
from scribe.routes.auth import auth_bp
from scribe.routes.export import export_bp
from scribe.routes.notes import notes_bp
from scribe.routes.milestones import milestones_bp
from scribe.routes.task_logs import task_logs_bp
from scribe.routes.projects import projects_bp
from scribe.routes.retrieval import retrieval_bp
from scribe.routes.settings import settings_bp
from scribe.routes.tasks import tasks_bp
from scribe.routes.groups import groups_bp
from scribe.routes.shares import shares_bp
from scribe.routes.in_app_notifications import notifications_bp
from scribe.routes.users import users_bp
from scribe.routes.api_keys import api_keys_bp
from scribe.routes.search import search_bp
from scribe.routes.profile import profile_bp
from scribe.routes.knowledge import knowledge_bp
from scribe.routes.rulebooks import rulebooks_bp
from scribe.routes.plugin import plugin_bp
from scribe.routes.design_systems import design_systems_bp
from scribe.routes.trash import trash_bp
from scribe.routes.dashboard import dashboard_bp
from scribe.routes.systems import systems_bp
from scribe.routes.canonical_systems import canonical_systems_bp
from scribe.routes.lessons import lessons_bp
from scribe.routes.snippets import snippets_bp
from scribe.routes.webhooks import webhooks_bp
from scribe.mcp import mount_mcp

STATIC_DIR = Path(__file__).parent / "static"
logger = logging.getLogger(__name__)


def create_app() -> Quart:
    # Configure logging
    logging.basicConfig(
        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
        level=getattr(logging, Config.LOG_LEVEL.upper(), logging.INFO),
    )

    # Validate config early so misconfigurations surface immediately with clear messages
    try:
        Config.validate()
    except ValueError as exc:
        logging.getLogger(__name__).error("Invalid configuration:\n%s", exc)
        raise

    app = Quart(__name__, static_folder=None)
    app.secret_key = Config.SECRET_KEY
    app.config["SESSION_COOKIE_HTTPONLY"] = True
    app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
    app.config["SESSION_COOKIE_SECURE"] = Config.SECURE_COOKIES

    if Config.SECRET_KEY == "dev-secret-change-me":
        logger.warning(
            "SECRET_KEY is set to the default value — session cookies are insecure. "
            "Set SECRET_KEY or SECRET_KEY_FILE for production use."
        )

    if not Config.TRUST_PROXY_HEADERS:
        logger.warning(
            "TRUST_PROXY_HEADERS is not set. If this instance is behind a reverse proxy "
            "(nginx, Caddy, Traefik) set TRUST_PROXY_HEADERS=true so rate limiting uses "
            "real client IPs rather than the proxy IP."
        )

    app.register_blueprint(admin_bp)
    app.register_blueprint(api)
    app.register_blueprint(auth_bp)
    app.register_blueprint(export_bp)
    app.register_blueprint(milestones_bp)
    app.register_blueprint(notes_bp)
    app.register_blueprint(projects_bp)
    app.register_blueprint(retrieval_bp)
    app.register_blueprint(settings_bp)
    app.register_blueprint(task_logs_bp)
    app.register_blueprint(tasks_bp)
    app.register_blueprint(groups_bp)
    app.register_blueprint(shares_bp)
    app.register_blueprint(notifications_bp)
    app.register_blueprint(users_bp)
    app.register_blueprint(api_keys_bp)
    app.register_blueprint(search_bp)
    app.register_blueprint(profile_bp)
    app.register_blueprint(knowledge_bp)
    app.register_blueprint(lessons_bp)
    app.register_blueprint(rulebooks_bp)
    app.register_blueprint(plugin_bp)
    app.register_blueprint(design_systems_bp)
    app.register_blueprint(trash_bp)
    app.register_blueprint(dashboard_bp)
    app.register_blueprint(systems_bp)
    app.register_blueprint(canonical_systems_bp)
    app.register_blueprint(snippets_bp)
    app.register_blueprint(webhooks_bp)

    @app.before_request
    async def before_request():
        g.request_start = time.monotonic()

    @app.after_request
    async def after_request(response):
        duration = time.monotonic() - getattr(g, "request_start", time.monotonic())
        duration_ms = round(duration * 1000, 1)
        # Downgrade noisy high-frequency / static paths to DEBUG
        _quiet = (
            request.path in {"/api/health", "/api/chat/status"}
            or request.path.startswith(("/static/", "/assets/", "/sw.js", "/manifest.json"))
        )
        log_fn = logger.debug if _quiet else logger.info
        log_fn(
            "%s %s %s %.1fms",
            request.method,
            request.path,
            response.status_code,
            duration_ms,
        )

        # Log usage for API requests (skip logs endpoint to avoid recursion)
        if request.path.startswith("/api/") and not request.path.startswith("/api/admin/logs"):
            try:
                from scribe.services.logging import log_usage

                user = getattr(g, "user", None)
                await log_usage(
                    user_id=user.id if user else None,
                    username=user.username if user else None,
                    endpoint=request.path,
                    method=request.method,
                    status_code=response.status_code,
                    duration_ms=duration_ms,
                )
            except Exception:
                logger.debug("Failed to log usage", exc_info=True)

        response.headers.setdefault("X-Content-Type-Options", "nosniff")
        response.headers.setdefault("X-Frame-Options", "DENY")
        response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
        response.headers.setdefault(
            "Content-Security-Policy",
            "default-src 'self'; "
            "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval'; "
            "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
            "img-src 'self' data: blob:; "
            "connect-src 'self'; "
            "font-src 'self' data: https://fonts.gstatic.com; "
            "object-src 'none'; "
            "base-uri 'self'; worker-src 'self' blob:;"
        )

        return response

    @app.before_serving
    async def startup():
        import asyncio

        # Set on the last line of this hook; the deferred backfill below waits
        # on it. An Event rather than a bare bool so the waiter is woken
        # instead of polling, and declared here — inside the hook, where
        # `asyncio` is in scope and a running loop exists — rather than in
        # `create_app`, which runs before either is true.
        _startup_finished = asyncio.Event()

        from scribe.services.auth import start_auth_token_retention_loop
        from scribe.services.embeddings import (
            backfill_milestone_embeddings, backfill_note_embeddings,
            backfill_rule_embeddings, backfill_system_embeddings,
        )
        from scribe.services.logging import start_log_retention_loop
        from scribe.services.notifications import start_notification_loop

        start_log_retention_loop()
        start_notification_loop()
        start_auth_token_retention_loop()

        # Write down any shipped default that moved in this release (#4225).
        #
        # INLINE AND AWAITED, deliberately, against the instinct the block
        # below argues for. What #4181 cost three hours was CONCURRENCY — a
        # background task created here racing this hook for the same pool.
        # Running sequentially creates no such contention, and this is twelve
        # single-row reads on an index against a table with a handful of rows.
        #
        # It has to finish before serving because the thing it protects is a
        # telemetry read: `band_hugs_floor` compares a band against a floor,
        # and across a release that changed the floor those describe different
        # regimes. A readout served before the change was recorded is the very
        # answer this exists to stop giving. Failure is logged and swallowed —
        # provenance is worth a lot and never worth refusing to boot.
        try:
            from scribe.services.retrieval_tuning import record_release_defaults
            moved = await record_release_defaults()
            for row in moved:
                if not row["baseline"]:
                    app.logger.info(
                        "release moved %s's shipped %s: %s -> %s",
                        row["surface"], row["dial"],
                        row["old_value"], row["new_value"],
                    )
        except Exception:
            app.logger.warning("could not record release defaults", exc_info=True)

        # Backfill embeddings for any notes that don't have one. Runs in the
        # background so it never blocks the server from accepting requests —
        # and, since #4181, not until the rest of this hook has finished.
        #
        # "Background" was true of REQUESTS and false of STARTUP. A task
        # created here begins immediately, while `before_serving` is still
        # running, so its four passes competed with the startup hook's own
        # database reads for the same connection pool. That is survivable on a
        # healthy disk and fatal on a sick one: on 2026-09-19 both this
        # backfill's first query and the hook's maintenance-hour read were
        # cancelled together when Hypercorn killed the worker at its lifespan
        # timeout, and the instance stayed down for three hours.
        #
        # The wait is on the SERVING FLAG rather than a sleep, because a sleep
        # would be a guess about how long the rest of the hook takes and would
        # be wrong in exactly the conditions that matter.
        async def _delayed_backfill() -> None:
            await _startup_finished.wait()
            try:
                await backfill_note_embeddings()
            except Exception:
                logger.warning("Embedding backfill failed", exc_info=True)
            # Rules got vectors in milestone 307; every rule written before it
            # has none, so this is the pass that makes them findable at all.
            try:
                await backfill_rule_embeddings()
            except Exception:
                logger.warning("Rule embedding backfill failed", exc_info=True)
            # Milestones got vectors in milestone 415, so a plan written before
            # it is findable only after this pass.
            try:
                await backfill_milestone_embeddings()
            except Exception:
                logger.warning("Milestone embedding backfill failed", exc_info=True)
            # Systems got vectors in #4251, so before this pass every charter
            # ever written is unfindable — a System could narrow a search and
            # never be the answer to one.
            try:
                await backfill_system_embeddings()
            except Exception:
                logger.warning("System embedding backfill failed", exc_info=True)
            # Snippets written before migration 0070 have no `notes.data` mirror,
            # and the location reverse lookup queries that column — an unfilled
            # row would read as "no snippet here" rather than as a gap. Separate
            # try block so neither backfill can skip the other.
            try:
                from scribe.services.snippets import backfill_snippet_data
                await backfill_snippet_data()
            except Exception:
                logger.warning("Snippet data backfill failed", exc_info=True)

        # Created here but gated on the flag released at the END of this hook,
        # so the task exists (nothing can forget to start it) while none of its
        # work overlaps startup's own.
        asyncio.create_task(_delayed_backfill())

        # RELEASED IN A `finally`, never after the work (rules 156 and 157).
        # The waiter above has no deadline of its own, and the way to make an
        # undeadlined wait safe is to make the release unmissable: if anything
        # below raises, the flag is still set and the task ends instead of
        # living on as a coroutine nobody will ever wake.
        try:
            # Recurrence scheduler (recurring-task spawn every 15m)
            from scribe.services.recurrence_scheduler import start_recurrence_scheduler
            start_recurrence_scheduler(asyncio.get_running_loop())

            # Version-pinning scheduler (daily auto-pin scan at 03:00 UTC)
            from scribe.services.version_pinning_scheduler import (
                start_version_pinning_scheduler,
            )
            start_version_pinning_scheduler(asyncio.get_running_loop())

            # Trash retention scheduler (daily expired-trash purge at 03:30 UTC)
            from scribe.services.trash_scheduler import start_trash_scheduler
            start_trash_scheduler(asyncio.get_running_loop())

            # DB maintenance scheduler (daily targeted VACUUM ANALYZE, default
            # 04:00 UTC). `get_maintenance_hour` is the FIRST database read in
            # this hook and is bounded for that reason (#4181) — see the long
            # comment above `_STARTUP_READ_TIMEOUT`. Anything added here that
            # touches the database needs the same treatment: a lifespan hook
            # that does not return is a worker that never serves.
            from scribe.services.db_maintenance_scheduler import (
                get_maintenance_hour,
                start_db_maintenance_scheduler,
            )
            start_db_maintenance_scheduler(
                asyncio.get_running_loop(), await get_maintenance_hour()
            )

            # Diagnostic instrumentation — heartbeat, signal handlers, asyncio
            # exception hook. Cheap (~1 log line/min), high diagnostic value when
            # the app crashes mysteriously. See services/diagnostics.py.
            from scribe.services.diagnostics import start_diagnostics
            start_diagnostics(asyncio.get_running_loop())
        finally:
            # STARTUP IS OVER — release the backfill (#4181). Everything above
            # is work the server needs done before it serves; everything the
            # backfill does is work that can wait for a server already up.
            logger.info("Startup complete; releasing deferred backfill")
            _startup_finished.set()

    @app.after_serving
    async def shutdown():
        from scribe.services.recurrence_scheduler import stop_recurrence_scheduler
        stop_recurrence_scheduler()
        from scribe.services.version_pinning_scheduler import (
            stop_version_pinning_scheduler,
        )
        stop_version_pinning_scheduler()
        from scribe.services.trash_scheduler import stop_trash_scheduler
        stop_trash_scheduler()
        from scribe.services.db_maintenance_scheduler import (
            stop_db_maintenance_scheduler,
        )
        stop_db_maintenance_scheduler()
        from scribe.services.diagnostics import stop_diagnostics
        stop_diagnostics()

    @app.route("/")
    async def serve_index():
        resp = await make_response(
            await send_from_directory(STATIC_DIR, "index.html")
        )
        resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
        return resp

    # File extensions that belong to the SPA or its assets.
    # Anything else with an extension is not a valid app path and gets a hard 404.
    _SPA_EXTENSIONS = {
        "", ".html", ".js", ".css", ".ico", ".png", ".jpg", ".jpeg",
        ".svg", ".webp", ".woff", ".woff2", ".ttf", ".otf", ".map", ".json",
    }

    @app.errorhandler(404)
    async def handle_404(error):
        # Return JSON 404 for API routes
        if request.path.startswith("/api/"):
            return jsonify({"error": "Not found"}), 404

        # Try to serve a real static file first
        path = request.path.lstrip("/")
        file_path = STATIC_DIR / path
        if path and file_path.is_file():
            return await send_from_directory(STATIC_DIR, path)

        # Reject paths with file extensions the app doesn't serve.
        # This turns scanner probes (.php, .asp, .cgi, etc.) into honest 404s
        # instead of serving them the Vue SPA with a misleading 200.
        from pathlib import PurePosixPath
        suffix = PurePosixPath(request.path).suffix.lower()
        if suffix not in _SPA_EXTENSIONS:
            return "", 404

        # SPA fallback for clean client-side routes (/notes/123, /chat, etc.)
        resp = await make_response(
            await send_from_directory(STATIC_DIR, "index.html")
        )
        resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
        return resp

    @app.errorhandler(500)
    async def handle_500(error):
        logger.exception("Internal server error on %s %s", request.method, request.path)

        try:
            from scribe.services.logging import log_error

            user = getattr(g, "user", None)
            await log_error(
                user_id=user.id if user else None,
                username=user.username if user else None,
                endpoint=request.path,
                method=request.method,
                error_type=type(error).__name__,
                error_message=str(error),
                traceback=tb_module.format_exc(),
            )
        except Exception:
            logger.debug("Failed to log error", exc_info=True)

        if request.path.startswith("/api/"):
            return jsonify({"error": "Internal server error"}), 500
        return "Internal Server Error", 500

    mount_mcp(app)
    return app