rename(547): the server is grimoire — package, image, DB defaults, backups

Step 2 of the Scribe -> Grimoire rename.

- src/scribe -> src/grimoire, and every import, patch target, module path,
  hypercorn entry point and `python -m grimoire.restore` with it; pyproject,
  uv.lock and the frontend package are named grimoire.
- Image git.fabledsword.com/bvandeusen/fabledgrimoire; CI's Postgres service
  runs as grimoire / grimoire_test.
- Compose files default the DB user, database and (prod) volume to grimoire,
  each overridable from .env, so an install created as Scribe keeps its data
  by setting POSTGRES_USER, POSTGRES_DB and PGDATA_VOLUME.
- Backups are written as grimoire-<ts>.tar with format grimoire-backup-tar.
  Archives named scribe-* with the scribe-backup-tar manifest still restore
  and are still counted at the destination (folder and S3), with tests.
- Exports, the MCP auth realm, the ASGI scope key and the persistent logger
  carry the new name.

Kept on purpose: the secret-sealing key label stays "scribe.secrets.v1",
because every stored secret was derived under it.

Plugin identity (plugin name, hooks, env vars, .scribe marker, skills) is
step 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/grimoire/routes/admin.py

@@ -0,0 +1,503 @@
+import asyncio
+import json
+
+from quart import Blueprint, Response, g, jsonify, request
+
+from grimoire.auth import admin_required, login_required, get_current_user_id
+from grimoire.services.auth import (
+    create_invitation,
+    delete_user,
+    is_registration_open,
+    list_pending_invitations,
+    list_users,
+    revoke_invitation,
+    set_registration_open,
+)
+from grimoire.services.backup import (
+    export_full_backup,
+    export_user_backup,
+    restore_full_backup,
+)
+from grimoire.services.email import SMTP_SETTING_KEYS, get_base_url, get_smtp_config, is_smtp_configured, send_test_email
+from grimoire.services.logging import get_logs, get_log_stats, log_audit
+from grimoire.services.notifications import send_invitation_email
+from grimoire.services.settings import (
+    SECRET_MASK,
+    get_admin_setting,
+    set_admin_setting,
+    set_setting,
+    set_settings_batch,
+)
+
+admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
+
+
+@admin_bp.route("/backup", methods=["GET"])
+@login_required
+async def backup():
+    uid = get_current_user_id()
+    scope = request.args.get("scope", "user")
+
+    if scope == "full":
+        # Full backup requires admin
+        if g.user.role != "admin":
+            return jsonify({"error": "Admin access required for full backup"}), 403
+        data = await export_full_backup()
+    else:
+        data = await export_user_backup(uid)
+
+    await log_audit("backup", user_id=uid, username=g.user.username, ip_address=request.remote_addr, details={"scope": scope})
+    return Response(
+        json.dumps(data, indent=2, default=str),
+        content_type="application/json",
+        headers={
+            "Content-Disposition": f'attachment; filename="fabled-backup-{scope}.json"',
+        },
+    )
+
+
+@admin_bp.route("/restore", methods=["POST"])
+@admin_required
+async def restore():
+    """Restore from an uploaded backup: the JSON download (application/json)
+    or the scheduled backup's tar (sent as the raw body). The upload is held
+    to the app's request-size limit; `python -m grimoire.restore` has none, and
+    is the way in for a bigger file or a fresh instance."""
+    if request.mimetype == "application/json":
+        data = await request.get_json()
+    else:
+        import io
+
+        from grimoire.services.backup_file import ArchiveError, read_archive
+
+        try:
+            data = read_archive(io.BytesIO(await request.get_data()))
+        except ArchiveError as exc:
+            return jsonify({"error": f"That file isn't a Grimoire backup: {exc}"}), 400
+    if not data:
+        return jsonify({"error": "No backup data provided"}), 400
+
+    stats = await restore_full_backup(data)
+    uid = get_current_user_id()
+    await log_audit("restore", user_id=uid, username=g.user.username, ip_address=request.remote_addr, details={"stats": stats})
+    return jsonify({"status": "ok", "stats": stats})
+
+
+@admin_bp.route("/users", methods=["GET"])
+@admin_required
+async def get_users():
+    users = await list_users()
+    return jsonify({"users": [u.to_dict() for u in users]})
+
+
+@admin_bp.route("/users/<int:user_id>", methods=["DELETE"])
+@admin_required
+async def remove_user(user_id: int):
+    current_uid = get_current_user_id()
+    if user_id == current_uid:
+        return jsonify({"error": "Cannot delete your own account"}), 400
+
+    deleted = await delete_user(user_id)
+    if not deleted:
+        return jsonify({"error": "User not found"}), 404
+    await log_audit("user_delete", user_id=current_uid, username=g.user.username, ip_address=request.remote_addr, details={"deleted_user_id": user_id})
+    return jsonify({"status": "ok"})
+
+
+@admin_bp.route("/registration", methods=["GET"])
+@admin_required
+async def get_registration():
+    open_status = await is_registration_open()
+    return jsonify({"open": open_status})
+
+
+@admin_bp.route("/registration", methods=["PUT"])
+@admin_required
+async def toggle_registration():
+    data = await request.get_json()
+    open_val = data.get("open")
+    if open_val is None:
+        return jsonify({"error": "Missing 'open' field"}), 400
+
+    uid = get_current_user_id()
+    await set_registration_open(uid, bool(open_val))
+    await log_audit("registration_toggle", user_id=uid, username=g.user.username, ip_address=request.remote_addr, details={"open": bool(open_val)})
+    return jsonify({"status": "ok", "open": bool(open_val)})
+
+
+@admin_bp.route("/smtp", methods=["GET"])
+@admin_required
+async def get_smtp():
+    config = await get_smtp_config()
+    # Mask password
+    if config.get("smtp_password"):
+        config["smtp_password"] = SECRET_MASK
+    return jsonify(config)
+
+
+@admin_bp.route("/smtp", methods=["PUT"])
+@admin_required
+async def update_smtp():
+    data = await request.get_json()
+    uid = get_current_user_id()
+
+    settings_to_save = {}
+    for key in SMTP_SETTING_KEYS:
+        if key in data:
+            # Skip password if it's the mask placeholder
+            if key == "smtp_password" and data[key] == SECRET_MASK:
+                continue
+            settings_to_save[key] = str(data[key])
+
+    if settings_to_save:
+        await set_settings_batch(uid, settings_to_save)
+    await log_audit("smtp_config", user_id=uid, username=g.user.username, ip_address=request.remote_addr)
+    return jsonify({"status": "ok"})
+
+
+@admin_bp.route("/smtp/test", methods=["POST"])
+@admin_required
+async def test_smtp():
+    data = await request.get_json()
+    recipient = (data.get("recipient") or "").strip()
+    if not recipient:
+        return jsonify({"error": "Recipient email is required"}), 400
+
+    uid = get_current_user_id()
+    try:
+        await send_test_email(recipient)
+        await log_audit("smtp_test", user_id=uid, username=g.user.username, ip_address=request.remote_addr, details={"recipient": recipient})
+        return jsonify({"status": "ok"})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+# The forge CONFIG is per-user keyring rows (Settings → Git forges); what
+# stays admin is the webhook secret, because the push endpoint
+# is one URL per instance and authenticates deliveries, not users.
+
+
+@admin_bp.route("/forge-webhook", methods=["GET"])
+@admin_required
+async def get_forge_webhook_settings():
+    from grimoire.config import Config
+    from grimoire.routes.webhooks import FORGE_WEBHOOK_SECRET_KEY
+
+    webhook_secret = (
+        await get_admin_setting(FORGE_WEBHOOK_SECRET_KEY, "")
+        or Config.FORGE_WEBHOOK_SECRET
+    )
+    return jsonify({
+        # Secrets never leave the server — the smtp_password convention:
+        # masked when set, empty when not.
+        "webhook_secret": SECRET_MASK if webhook_secret else "",
+    })
+
+
+@admin_bp.route("/forge-webhook", methods=["PUT"])
+@admin_required
+async def update_forge_webhook_settings():
+    from grimoire.routes.webhooks import FORGE_WEBHOOK_SECRET_KEY
+
+    data = await request.get_json() or {}
+    uid = get_current_user_id()
+    webhook_secret = data.get("webhook_secret")
+    # The mask coming back means "unchanged" — the form round-trips what GET
+    # showed it, and storing the mask would silently break the integration.
+    if webhook_secret is not None and webhook_secret != SECRET_MASK:
+        await set_admin_setting(FORGE_WEBHOOK_SECRET_KEY, str(webhook_secret))
+    # The secret is deliberately absent from the audit detail.
+    await log_audit(
+        "forge_webhook_config", user_id=uid, username=g.user.username,
+        ip_address=request.remote_addr, details={},
+    )
+    return jsonify({"status": "ok"})
+
+
+@admin_bp.route("/logs", methods=["GET"])
+@admin_required
+async def list_logs():
+    category = request.args.get("category")
+    user_id = request.args.get("user_id", type=int)
+    search = request.args.get("search")
+    date_from = request.args.get("date_from")
+    date_to = request.args.get("date_to")
+    limit = request.args.get("limit", 50, type=int)
+    offset = request.args.get("offset", 0, type=int)
+
+    logs, total = await get_logs(
+        category=category,
+        user_id=user_id,
+        search=search,
+        date_from=date_from,
+        date_to=date_to,
+        limit=min(limit, 200),
+        offset=offset,
+    )
+    return jsonify({"logs": logs, "total": total})
+
+
+@admin_bp.route("/logs/stats", methods=["GET"])
+@admin_required
+async def log_stats():
+    stats = await get_log_stats()
+    return jsonify(stats)
+
+
+@admin_bp.route("/base-url", methods=["GET"])
+@admin_required
+async def get_base_url_setting():
+    base_url = await get_base_url()
+    return jsonify({"base_url": base_url})
+
+
+@admin_bp.route("/base-url", methods=["PUT"])
+@admin_required
+async def update_base_url():
+    data = await request.get_json()
+    url = (data.get("base_url") or "").strip().rstrip("/")
+    if url:
+        scheme = url.split("://")[0].lower() if "://" in url else ""
+        if scheme not in ("http", "https"):
+            return jsonify({"error": "Base URL must use http or https"}), 400
+    uid = get_current_user_id()
+    await set_setting(uid, "base_url", url)
+    await log_audit("base_url_config", user_id=uid, username=g.user.username, ip_address=request.remote_addr, details={"base_url": url})
+    return jsonify({"status": "ok"})
+
+
+@admin_bp.route("/db-maintenance", methods=["GET"])
+@admin_required
+async def get_db_maintenance():
+    """Current DB-maintenance config + the last run's summary."""
+    from grimoire.services.db_maintenance import get_last_run
+    from grimoire.services.db_maintenance_scheduler import (
+        get_maintenance_hour,
+        is_maintenance_enabled,
+    )
+    return jsonify({
+        "enabled": await is_maintenance_enabled(),
+        "hour": await get_maintenance_hour(),
+        "last_run": await get_last_run(),
+    })
+
+
+@admin_bp.route("/db-maintenance/health", methods=["GET"])
+@admin_required
+async def get_db_maintenance_health():
+    """Read-only per-table bloat/health stats from Postgres + total DB size."""
+    from grimoire.services.db_maintenance import get_table_health
+    return jsonify(await get_table_health())
+
+
+@admin_bp.route("/db-maintenance", methods=["PUT"])
+@admin_required
+async def update_db_maintenance():
+    """Set whether scheduled maintenance runs and at what UTC hour."""
+    from grimoire.services.db_maintenance_scheduler import reschedule_db_maintenance
+    data = await request.get_json() or {}
+    enabled = bool(data.get("enabled", True))
+    try:
+        hour = int(data.get("hour", 4))
+    except (TypeError, ValueError):
+        return jsonify({"error": "hour must be an integer 0–23"}), 400
+    if not 0 <= hour <= 23:
+        return jsonify({"error": "hour must be between 0 and 23"}), 400
+
+    await set_admin_setting("db_maintenance_enabled", "true" if enabled else "false")
+    await set_admin_setting("db_maintenance_hour", str(hour))
+    reschedule_db_maintenance(hour)
+    uid = get_current_user_id()
+    await log_audit(
+        "db_maintenance_config", user_id=uid, username=g.user.username,
+        ip_address=request.remote_addr, details={"enabled": enabled, "hour": hour},
+    )
+    return jsonify({"status": "ok", "enabled": enabled, "hour": hour})
+
+
+@admin_bp.route("/db-maintenance/run", methods=["POST"])
+@admin_required
+async def run_db_maintenance_now():
+    """Run a VACUUM (ANALYZE) sweep immediately and return its summary."""
+    from grimoire.services.db_maintenance import run_maintenance
+    uid = get_current_user_id()
+    await log_audit(
+        "db_maintenance_run", user_id=uid, username=g.user.username,
+        ip_address=request.remote_addr,
+    )
+    summary = await run_maintenance()
+    return jsonify(summary)
+
+
+# ── Scheduled backups ────────────────────────────────────────────────────────
+# One dated, uncompressed tar per run to a folder or an S3 bucket — see
+# services/backup_file.py for why it is shaped that way, backup_destination.py
+# for where it goes, backup_job.py for when.
+
+# How long "Test" waits on the destination. A folder on a hung NFS mount has
+# no timeout of its own, and the request must come back either way.
+_BACKUP_CHECK_TIMEOUT = 45.0
+
+
+async def _backups_payload() -> dict:
+    from datetime import datetime, timezone
+
+    from grimoire.services import backup_destination as dest
+    from grimoire.services import backup_job as job
+
+    config = await dest.load_config()
+    secret, stored = await dest.load_secret()
+    runs = await job.list_runs()
+    return {
+        "config": config.to_dict(),
+        # Never the secret itself: whether one is kept, and whether this
+        # instance can still open it (it can't after a restore onto another
+        # SECRET_KEY, and then it has to be entered again).
+        "secret": {"set": stored, "readable": secret is not None},
+        "problems": config.problems(has_secret=secret is not None),
+        "running": job.is_running(),
+        "runs": runs,
+        **job.summarise(runs, enabled=config.enabled, now=datetime.now(timezone.utc)),
+    }
+
+
+@admin_bp.route("/backups", methods=["GET"])
+@admin_required
+async def get_backups():
+    """The backup destination, its schedule, and the recent runs."""
+    return jsonify(await _backups_payload())
+
+
+@admin_bp.route("/backups", methods=["PUT"])
+@admin_required
+async def update_backups():
+    """Change the destination or schedule. `s3_secret` replaces the stored
+    secret when it is sent and isn't the mask; `clear_s3_secret` removes it."""
+    from grimoire.services import backup_destination as dest
+
+    data = await request.get_json() or {}
+    try:
+        config = dest.apply_changes(await dest.load_config(), data)
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 400
+    await dest.save_config(config)
+    secret = data.get("s3_secret")
+    if data.get("clear_s3_secret"):
+        await dest.save_secret("")
+    elif isinstance(secret, str) and secret and secret != SECRET_MASK:
+        await dest.save_secret(secret)
+    uid = get_current_user_id()
+    await log_audit(
+        "backup_config", user_id=uid, username=g.user.username,
+        ip_address=request.remote_addr,
+        details={**config.to_dict(), "secret_changed": bool(
+            data.get("clear_s3_secret") or (isinstance(secret, str) and secret and secret != SECRET_MASK)
+        )},
+    )
+    return jsonify(await _backups_payload())
+
+
+@admin_bp.route("/backups/check", methods=["POST"])
+@admin_required
+async def check_backups_destination():
+    """Ask the destination whether it would take a backup, without sending
+    one. The body may carry unsaved form values, so a destination can be
+    tested before it is saved; the stored secret is used unless one is sent."""
+    from grimoire.services import backup_destination as dest
+
+    data = await request.get_json() or {}
+    try:
+        config = dest.apply_changes(await dest.load_config(), data)
+    except ValueError as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+    secret = data.get("s3_secret")
+    if not (isinstance(secret, str) and secret and secret != SECRET_MASK):
+        secret, _ = await dest.load_secret()
+    try:
+        message = await asyncio.wait_for(
+            dest.check_destination(config, secret), _BACKUP_CHECK_TIMEOUT,
+        )
+    except dest.DestinationError as exc:
+        return jsonify({"ok": False, "message": str(exc)})
+    except (TimeoutError, asyncio.TimeoutError):
+        return jsonify({"ok": False, "message": (
+            f"The destination didn't answer within {int(_BACKUP_CHECK_TIMEOUT)} seconds."
+        )})
+    return jsonify({"ok": True, "message": message})
+
+
+@admin_bp.route("/backups/run", methods=["POST"])
+@admin_required
+async def run_backup_now():
+    """Start a backup now. It runs in the background; the page follows it
+    through GET /backups (`running`, then the newest run)."""
+    from grimoire.services import backup_job as job
+
+    try:
+        job.start_backup_now()
+    except job.BackupBusy:
+        return jsonify({"error": "A backup is already running."}), 409
+    uid = get_current_user_id()
+    await log_audit("backup_run", user_id=uid, username=g.user.username,
+                    ip_address=request.remote_addr, details={})
+    return jsonify({"status": "started"}), 202
+
+
+@admin_bp.route("/invitations", methods=["POST"])
+@admin_required
+async def create_invite():
+    data = await request.get_json()
+    email = (data.get("email") or "").strip().lower()
+    if not email:
+        return jsonify({"error": "Email is required"}), 400
+
+    if not await is_smtp_configured():
+        return jsonify({"error": "SMTP is not configured. Configure email settings first."}), 400
+
+    uid = get_current_user_id()
+    raw_token = await create_invitation(email, uid)
+    base_url = await get_base_url()
+    invite_url = f"{base_url}/register-invite?token={raw_token}"
+    asyncio.create_task(send_invitation_email(email, invite_url, g.user.username))
+    await log_audit(
+        "invitation_created",
+        user_id=uid,
+        username=g.user.username,
+        ip_address=request.remote_addr,
+        details={"invited_email": email},
+    )
+    return jsonify({"status": "ok"}), 201
+
+
+@admin_bp.route("/invitations", methods=["GET"])
+@admin_required
+async def get_invitations():
+    invitations = await list_pending_invitations()
+    return jsonify({
+        "invitations": [
+            {
+                "id": inv.id,
+                "email": inv.email,
+                "created_at": inv.created_at.isoformat(),
+                "expires_at": inv.expires_at.isoformat(),
+            }
+            for inv in invitations
+        ]
+    })
+
+
+@admin_bp.route("/invitations/<int:invitation_id>", methods=["DELETE"])
+@admin_required
+async def delete_invitation(invitation_id: int):
+    uid = get_current_user_id()
+    revoked = await revoke_invitation(invitation_id)
+    if not revoked:
+        return jsonify({"error": "Invitation not found or already used"}), 404
+    await log_audit(
+        "invitation_revoked",
+        user_id=uid,
+        username=g.user.username,
+        ip_address=request.remote_addr,
+        details={"invitation_id": invitation_id},
+    )
+    return jsonify({"status": "ok"})