rename(547): the server is grimoire — package, image, DB defaults, backups

Step 2 of the Scribe -> Grimoire rename.

- src/scribe -> src/grimoire, and every import, patch target, module path,
  hypercorn entry point and `python -m grimoire.restore` with it; pyproject,
  uv.lock and the frontend package are named grimoire.
- Image git.fabledsword.com/bvandeusen/fabledgrimoire; CI's Postgres service
  runs as grimoire / grimoire_test.
- Compose files default the DB user, database and (prod) volume to grimoire,
  each overridable from .env, so an install created as Scribe keeps its data
  by setting POSTGRES_USER, POSTGRES_DB and PGDATA_VOLUME.
- Backups are written as grimoire-<ts>.tar with format grimoire-backup-tar.
  Archives named scribe-* with the scribe-backup-tar manifest still restore
  and are still counted at the destination (folder and S3), with tests.
- Exports, the MCP auth realm, the ASGI scope key and the persistent logger
  carry the new name.

Kept on purpose: the secret-sealing key label stays "scribe.secrets.v1",
because every stored secret was derived under it.

Plugin identity (plugin name, hooks, env vars, .scribe marker, skills) is
step 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/grimoire/routes/rulebooks.py

@@ -0,0 +1,503 @@
+"""Rulebook / topic REST endpoints.
+
+Wraps services/rulebooks.py. Standard Grimoire auth: get_current_user_id() is the
+authenticated owner; the service enforces ownership scoping.
+"""
+from __future__ import annotations
+
+from quart import Blueprint, jsonify, request
+
+from grimoire.auth import get_current_user_id, login_required
+import grimoire.services.moments as moments_svc
+import grimoire.services.rulebooks as rulebooks_svc
+from grimoire.services.trash import delete as trash_delete
+from grimoire.services.rule_usage import (
+    empty_rule_usage, record_rule_pulled, usage_for_rules,
+)
+
+rulebooks_bp = Blueprint("rulebooks", __name__, url_prefix="/api")
+
+
+def _moments_or_refusal(data: dict):
+    """The request's `moments`, validated BEFORE any write — so an unknown
+    name is a 400 and not a rule created without the mounts it asked for.
+    Absent means "leave them alone" (None), as on the MCP door."""
+    try:
+        return moments_svc.require_moments(data.get("moments")), None
+    except ValueError as exc:
+        return None, (jsonify({"error": str(exc)}), 400)
+
+
+# ── Rulebooks ───────────────────────────────────────────────────────────
+
+@rulebooks_bp.get("/rulebooks")
+@login_required
+async def list_rulebooks():
+    rows = await rulebooks_svc.list_rulebooks(get_current_user_id())
+    return jsonify({"rulebooks": [rb.to_dict() for rb in rows]})
+
+
+@rulebooks_bp.post("/rulebooks")
+@login_required
+async def create_rulebook():
+    data = await request.get_json() or {}
+    title = (data.get("title") or "").strip()
+    if not title:
+        return jsonify({"error": "title is required"}), 400
+    rb = await rulebooks_svc.create_rulebook(
+        user_id=get_current_user_id(),
+        title=title,
+        description=data.get("description", ""),
+    )
+    return jsonify(rb.to_dict()), 201
+
+
+@rulebooks_bp.get("/rulebooks/<int:rulebook_id>")
+@login_required
+async def get_rulebook(rulebook_id: int):
+    rb = await rulebooks_svc.get_rulebook(rulebook_id, get_current_user_id())
+    if rb is None:
+        return jsonify({"error": "rulebook not found"}), 404
+    return jsonify(rb.to_dict())
+
+
+@rulebooks_bp.patch("/rulebooks/<int:rulebook_id>")
+@login_required
+async def update_rulebook(rulebook_id: int):
+    data = await request.get_json() or {}
+    fields = {k: v for k, v in data.items() if k in ("title", "description")}
+    rb = await rulebooks_svc.update_rulebook(rulebook_id, get_current_user_id(), **fields)
+    if rb is None:
+        return jsonify({"error": "rulebook not found"}), 404
+    return jsonify(rb.to_dict())
+
+
+@rulebooks_bp.delete("/rulebooks/<int:rulebook_id>")
+@login_required
+async def delete_rulebook(rulebook_id: int):
+    await trash_delete(get_current_user_id(), "rulebook", rulebook_id)
+    return "", 204
+
+
+# ── Topics ──────────────────────────────────────────────────────────────
+
+@rulebooks_bp.get("/rulebooks/<int:rulebook_id>/topics")
+@login_required
+async def list_topics(rulebook_id: int):
+    try:
+        rows = await rulebooks_svc.list_topics(rulebook_id, get_current_user_id())
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 404
+    return jsonify({"topics": [t.to_dict() for t in rows]})
+
+
+@rulebooks_bp.post("/rulebooks/<int:rulebook_id>/topics")
+@login_required
+async def create_topic(rulebook_id: int):
+    data = await request.get_json() or {}
+    title = (data.get("title") or "").strip()
+    if not title:
+        return jsonify({"error": "title is required"}), 400
+    try:
+        topic = await rulebooks_svc.create_topic(
+            rulebook_id=rulebook_id,
+            user_id=get_current_user_id(),
+            title=title,
+            description=data.get("description", ""),
+            order_index=data.get("order_index", 0),
+        )
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 404
+    return jsonify(topic.to_dict()), 201
+
+
+@rulebooks_bp.patch("/rulebook-topics/<int:topic_id>")
+@login_required
+async def update_topic(topic_id: int):
+    data = await request.get_json() or {}
+    fields = {
+        k: v for k, v in data.items()
+        if k in ("title", "description", "order_index")
+    }
+    topic = await rulebooks_svc.update_topic(topic_id, get_current_user_id(), **fields)
+    if topic is None:
+        return jsonify({"error": "topic not found"}), 404
+    return jsonify(topic.to_dict())
+
+
+@rulebooks_bp.delete("/rulebook-topics/<int:topic_id>")
+@login_required
+async def delete_topic(topic_id: int):
+    if await trash_delete(get_current_user_id(), "topic", topic_id) is None:
+        return jsonify({"error": "topic not found"}), 404
+    return "", 204
+
+
+# ── Rules ───────────────────────────────────────────────────────────────
+
+@rulebooks_bp.get("/rules")
+@login_required
+async def list_rules():
+    def _opt_int(name):
+        raw = request.args.get(name)
+        return int(raw) if raw else None
+
+    try:
+        rulebook_id = _opt_int("rulebook_id")
+        topic_id = _opt_int("topic_id")
+        project_id = _opt_int("project_id")
+    except ValueError:
+        return jsonify({"error": "rulebook_id, topic_id, project_id must be integers"}), 400
+
+    uid = get_current_user_id()
+    rows = await rulebooks_svc.list_rules(
+        user_id=uid,
+        rulebook_id=rulebook_id,
+        topic_id=topic_id,
+        project_id=project_id,
+    )
+    items = [r.to_dict() for r in rows]
+    # One aggregate for the whole page — a per-row lookup here would be N+1 by
+    # construction, the same reason the snippet list does it this way. Every
+    # row gets the key, zero-filled, so the UI renders "never surfaced" rather
+    # than having to treat a missing field as a state. That matters more here
+    # than for snippets: every rule on every install predates this table, so
+    # for a while the zero-filled shape IS the common case.
+    usage = await usage_for_rules([int(it["id"]) for it in items])
+    # Zero-filled like `usage`, for the same reason: the UI renders "mounted
+    # nowhere" from an empty list rather than treating a missing key as a state.
+    mounted = await rulebooks_svc.list_rule_moments([int(it["id"]) for it in items])
+    for it in items:
+        it["usage"] = usage.get(int(it["id"]), empty_rule_usage())
+        it["moments"] = mounted.get(int(it["id"]), [])
+    return jsonify({"rules": items})
+
+
+@rulebooks_bp.post("/rulebook-topics/<int:topic_id>/rules")
+@login_required
+async def create_rule(topic_id: int):
+    data = await request.get_json() or {}
+    title = (data.get("title") or "").strip()
+    statement = (data.get("statement") or "").strip()
+    if not title or not statement:
+        return jsonify({"error": "title and statement are required"}), 400
+    moments, refused = _moments_or_refusal(data)
+    if refused:
+        return refused
+    try:
+        rule = await rulebooks_svc.create_rule(
+            topic_id=topic_id,
+            user_id=get_current_user_id(),
+            title=title,
+            statement=statement,
+            why=data.get("why", ""),
+            how_to_apply=data.get("how_to_apply", ""),
+            order_index=data.get("order_index", 0),
+            when_to_apply=data.get("when_to_apply", ""),
+            # The human door carries `kind` too, and without the MCP door's
+            # required provenance: an operator editing their own preference
+            # owes nobody an explanation. That requirement is about auditing
+            # what the AGENT changed, not what they did themselves.
+            kind=data.get("kind", "rule"),
+            arose_from_id=data.get("arose_from_id", 0) or 0,
+            verify_with=data.get("verify_with", ""),
+            expires_when=data.get("expires_when", ""),
+        )
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 404
+    return jsonify(await rulebooks_svc.rule_detail(
+        get_current_user_id(), rule, data.get("system_ids"), moments,
+    )), 201
+
+
+@rulebooks_bp.get("/rules/<int:rule_id>")
+@login_required
+async def get_rule(rule_id: int):
+    uid = get_current_user_id()
+    rule = await rulebooks_svc.get_rule(rule_id, uid)
+    if rule is None:
+        return jsonify({"error": "rule not found"}), 404
+    # `rest_` rather than `mcp_`, and the prefix is load-bearing: "is this rule
+    # dead weight?" is served by any pull, but "did that injected hint land?"
+    # — the question this arm exists to answer — is served by AGENT pulls only.
+    # A person clicking through the rule list says nothing about the hint.
+    record_rule_pulled(user_id=uid, rule_id=int(rule.id), source="rest_rule")
+    return jsonify(await rulebooks_svc.rule_detail(uid, rule))
+
+
+@rulebooks_bp.patch("/rules/<int:rule_id>")
+@login_required
+async def update_rule(rule_id: int):
+    data = await request.get_json() or {}
+    uid = get_current_user_id()
+    fields = {
+        k: v for k, v in data.items()
+        if k in ("title", "statement", "why", "how_to_apply", "order_index",
+                 "when_to_apply", "kind", "arose_from_id",
+                 "verify_with", "expires_when")
+    }
+    # No clear_fields here: a form sends "" for an emptied input, and the
+    # service normalises "" to NULL for every nullable text column. The MCP
+    # door needs the explicit list only because "" already means "unchanged"
+    # there — two idioms, one outcome.
+    moments, refused = _moments_or_refusal(data)
+    if refused:
+        return refused
+    rule = await rulebooks_svc.update_rule(rule_id, uid, **fields)
+    if rule is None:
+        return jsonify({"error": "rule not found"}), 404
+    return jsonify(await rulebooks_svc.rule_detail(
+        uid, rule, data.get("system_ids"), moments,
+    ))
+
+
+@rulebooks_bp.get("/rules/<int:rule_id>/versions")
+@login_required
+async def list_rule_versions(rule_id: int):
+    """A rule's edit history, newest first.
+
+    Listing form only — a rule's `statement` and `why` run to thousands of
+    characters, so a history list carrying every field would be unreadable
+    and expensive to send. Open one for the text.
+    """
+    uid = get_current_user_id()
+    versions = await rulebooks_svc.list_rule_versions(rule_id, uid)
+    if versions is None:
+        return jsonify({"error": "rule not found"}), 404
+    return jsonify({
+        "versions": [v.to_dict(include_text=False) for v in versions],
+    })
+
+
+@rulebooks_bp.get("/rules/<int:rule_id>/versions/<int:version_id>")
+@login_required
+async def get_rule_version(rule_id: int, version_id: int):
+    """One snapshot in full — what the rule said before that edit."""
+    uid = get_current_user_id()
+    version = await rulebooks_svc.get_rule_version(rule_id, version_id, uid)
+    if version is None:
+        return jsonify({"error": "version not found"}), 404
+    return jsonify(version.to_dict(include_text=True))
+
+
+# NO restore route FOR A RULE, deliberately. A note version
+# can be restored; a binding instruction should not be revertible in one
+# click. Putting a rewrite back goes through update_rule, which takes its own
+# snapshot and leaves the undo in the history like any other edit — a silent
+# revert would erase the only record of why the rewrite happened.
+#
+# A PREFERENCE IS THE EXCEPTION, and the route below refuses anything else.
+# The reasoning above turns on the rewrite being the operator's own decision.
+# A preference's rewrite is not: the agent makes it mid-work without asking,
+# which is what the kind is for. Reverting one is a veto over someone else's
+# edit rather than an undo of your own, and making that veto expensive leaves
+# drift supervised in name only. The safeguard survives intact, because the restore goes through
+# update_rule too: the rewrite stays in the history with the revert recorded
+# after it, so the history gains an entry rather than losing one.
+
+
+@rulebooks_bp.get("/rules/drift")
+@login_required
+async def preference_drift():
+    """Preferences that have been rewritten, most recently changed first.
+
+    One row per preference carrying its latest rewrite, what it said before,
+    and the record that taught the change. `?limit=` caps the list.
+    """
+    uid = get_current_user_id()
+    try:
+        limit = int(request.args.get("limit", 20))
+    except (TypeError, ValueError):
+        limit = 20
+    rows = await rulebooks_svc.recent_preference_drift(uid, limit=limit)
+    return jsonify({"drift": rows})
+
+
+@rulebooks_bp.post("/rules/<int:rule_id>/versions/<int:version_id>/restore")
+@login_required
+async def restore_rule_version(rule_id: int, version_id: int):
+    """Put a preference back to what that version said. Preferences only.
+
+    409 rather than 400 on a rule: the request is well-formed and the caller
+    is not wrong to have asked — this rule is simply in a state where the
+    action does not apply, and the message says which state and why.
+    """
+    uid = get_current_user_id()
+    try:
+        rule = await rulebooks_svc.restore_rule_version(rule_id, version_id, uid)
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 409
+    if rule is None:
+        return jsonify({"error": "rule or version not found"}), 404
+    return jsonify(await rulebooks_svc.rule_detail(uid, rule))
+
+
+@rulebooks_bp.post("/rules/<int:rule_id>/relations")
+@login_required
+async def relate_rules(rule_id: int):
+    """Draw a typed edge FROM this rule to another.
+
+    Body: {"to_rule_id": N, "kind": "co_surfaces"|"overrides"|"elaborates",
+    "note": "..."}. Idempotent — re-drawing an edge returns the existing one.
+    """
+    data = await request.get_json() or {}
+    to_rule_id = data.get("to_rule_id")
+    if not isinstance(to_rule_id, int):
+        return jsonify({"error": "to_rule_id is required"}), 400
+    try:
+        relation = await rulebooks_svc.add_rule_relation(
+            get_current_user_id(), rule_id, to_rule_id,
+            data.get("kind", ""), data.get("note", ""),
+        )
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 400
+    if relation is None:
+        return jsonify({"error": "rule not found"}), 404
+    return jsonify(relation.to_dict()), 201
+
+
+@rulebooks_bp.delete("/rule-relations/<int:relation_id>")
+@login_required
+async def unrelate_rules(relation_id: int):
+    if not await rulebooks_svc.remove_rule_relation(get_current_user_id(), relation_id):
+        return jsonify({"error": "relation not found"}), 404
+    return "", 204
+
+
+@rulebooks_bp.post("/rules/<int:rule_id>/move")
+@login_required
+async def move_rule(rule_id: int):
+    """Give a rule a new home: {topic_id} makes it global, {project_id} makes
+    it that project's. The MCP twin is move_rule, under the same names."""
+    data = await request.get_json() or {}
+    uid = get_current_user_id()
+    try:
+        rule = await rulebooks_svc.move_rule(
+            rule_id, uid,
+            topic_id=int(data.get("topic_id") or 0),
+            project_id=int(data.get("project_id") or 0),
+        )
+    except (TypeError, ValueError) as exc:
+        msg = str(exc)
+        return jsonify({"error": msg}), 404 if "not found" in msg else 400
+    if rule is None:
+        return jsonify({"error": "rule not found"}), 404
+    return jsonify(await rulebooks_svc.rule_detail(uid, rule))
+
+
+@rulebooks_bp.delete("/rules/<int:rule_id>")
+@login_required
+async def delete_rule(rule_id: int):
+    if await trash_delete(get_current_user_id(), "rule", rule_id) is None:
+        return jsonify({"error": "rule not found"}), 404
+    return "", 204
+
+
+# ── A project's rule listing ───────────────────────────────────────────
+
+@rulebooks_bp.get("/projects/<int:project_id>/rules")
+@login_required
+async def get_project_rules(project_id: int):
+    result = await rulebooks_svc.get_applicable_rules(
+        project_id=project_id, user_id=get_current_user_id(),
+    )
+    return jsonify(result)
+
+
+@rulebooks_bp.post("/projects/<int:project_id>/rules")
+@login_required
+async def create_project_rule(project_id: int):
+    """Create a rule scoped to a single project. Frontend fast path."""
+    data = await request.get_json() or {}
+    statement = (data.get("statement") or "").strip()
+    if not statement:
+        return jsonify({"error": "statement is required"}), 400
+    # Checked here as well as in the service, only for the STATUS CODE: the
+    # service raises ValueError, which this route maps to 404 for "project not
+    # found", and a missing trigger is a 400. The service stays the guard —
+    # this is the door telling the truth about whose mistake it was.
+    if not (data.get("when_to_apply") or "").strip():
+        return jsonify({
+            "error": "when_to_apply is required: a rule with no trigger never "
+                     "surfaces at the moment it applies."
+        }), 400
+    title = (data.get("title") or "").strip() or statement.split(".")[0][:50]
+    moments, refused = _moments_or_refusal(data)
+    if refused:
+        return refused
+    try:
+        rule = await rulebooks_svc.create_project_rule(
+            project_id=project_id,
+            user_id=get_current_user_id(),
+            title=title,
+            statement=statement,
+            why=data.get("why", ""),
+            how_to_apply=data.get("how_to_apply", ""),
+            order_index=data.get("order_index", 0),
+            when_to_apply=data.get("when_to_apply", ""),
+            # The human door carries `kind` too, and without the MCP door's
+            # required provenance: an operator editing their own preference
+            # owes nobody an explanation. That requirement is about auditing
+            # what the AGENT changed, not what they did themselves.
+            kind=data.get("kind", "rule"),
+            arose_from_id=data.get("arose_from_id", 0) or 0,
+            verify_with=data.get("verify_with", ""),
+            expires_when=data.get("expires_when", ""),
+        )
+    except ValueError as exc:
+        return jsonify({"error": str(exc)}), 404
+    return jsonify(await rulebooks_svc.rule_detail(
+        get_current_user_id(), rule, data.get("system_ids"), moments,
+    )), 201
+
+
+# ── The staleness sweep ────────────────────────────────────────────────
+
+@rulebooks_bp.get("/rules-due-for-verification")
+@login_required
+async def rules_due_for_verification():
+    """Rules that carry a check, oldest verification first, never-checked top.
+
+    Query params: older_than_days, never_only. A rule with no
+    `verify_with` never appears — it is a decision, not a fact.
+    """
+    uid = get_current_user_id()
+    args = request.args
+    try:
+        older = int(args.get("older_than_days", 0) or 0)
+    except ValueError:
+        return jsonify({"error": "older_than_days must be an integer"}), 400
+    try:
+        rules = await rulebooks_svc.rules_due_for_verification(
+            uid,
+            older_than_days=older,
+            never_only=args.get("never_only", "").lower() in ("1", "true", "yes"),
+        )
+    except ValueError as exc:
+        # An unrecognised tier is a 400, not a silently narrowed result set:
+        # a filter that quietly answers a different question is the failure
+        # this whole surface exists to catch.
+        return jsonify({"error": str(exc)}), 400
+    return jsonify({
+        "rules": [rulebooks_svc.verification_row(r) for r in rules],
+        "total": len(rules),
+    })
+
+
+@rulebooks_bp.post("/rules/<int:rule_id>/verify")
+@login_required
+async def mark_rule_verified(rule_id: int):
+    """Record that the rule's check was run. Body: {"still_true": bool}.
+
+    `still_true: false` writes nothing — a rule whose check failed is wrong,
+    not in a recordable state — so it stays at the top of the sweep.
+    """
+    data = await request.get_json() or {}
+    uid = get_current_user_id()
+    still_true = data.get("still_true", True)
+    rule = await rulebooks_svc.mark_rule_verified(rule_id, uid, bool(still_true))
+    if rule is None:
+        return jsonify({"error": "rule not found, or carries no verify_with"}), 404
+    payload = await rulebooks_svc.rule_detail(uid, rule)
+    payload["verified"] = bool(still_true)
+    return jsonify(payload)