feat(backup): v3 backup covers rulebooks, rules, events + join tables

The v2 backup silently dropped the entire rulebook system (rulebooks, topics,
rules), the project subscription/suppression join tables, and events — so a
'full' backup wasn't. v3 adds all of them with FK re-mapping on restore, and a
_not_included field that names the still-deferred tables (ACL groups/shares,
api_keys, embeddings, transient/operational) so the gap is explicit, not silent.

restore_full_backup routes v2 and v3 through one path; v3-only sections are
guarded by data.get so a v2 payload still restores cleanly.

Tests: version/coverage constants, pure join-table row helpers, and the export
contract via a mocked session (CI has no DB; full round-trip is a manual check).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/scribe/services/backup.py

@@ -1,20 +1,44 @@
 import logging
 from datetime import date, datetime, timezone
 
-from sqlalchemy import select
+from sqlalchemy import or_, select
 
 from scribe.models import async_session
+from scribe.models.event import Event
 from scribe.models.milestone import Milestone
 from scribe.models.note import Note
 from scribe.models.note_draft import NoteDraft
 from scribe.models.note_version import NoteVersion
 from scribe.models.project import Project
+from scribe.models.rulebook import (
+    Rule,
+    Rulebook,
+    RulebookTopic,
+    project_rule_suppressions,
+    project_rulebook_subscriptions,
+    project_topic_suppressions,
+)
 from scribe.models.setting import Setting
 from scribe.models.task_log import TaskLog
 from scribe.models.user import User
 
 logger = logging.getLogger(__name__)
 
+# Backup format version. v3 (2026-06) added rulebooks/topics/rules + their
+# project subscription/suppression join tables, and events — all silently
+# dropped by v2. Bump when the serialized schema changes.
+BACKUP_VERSION = 3
+
+# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
+# explicit rather than silent. ACL (groups/shares) is a coherent follow-up;
+# embeddings are derived (regenerated from note bodies); api_keys are sensitive
+# credentials; the rest are transient/operational.
+_NOT_INCLUDED = [
+    "groups", "group_memberships", "project_shares", "note_shares",
+    "api_keys", "embeddings", "app_logs", "notifications", "invitations",
+    "password_resets", "user_profiles",
+]
+
 
 def _dt(val: str | None) -> datetime:
     return datetime.fromisoformat(val) if val else datetime.now(timezone.utc)
@@ -24,12 +48,24 @@ def _d(val: str | None) -> date | None:
     return date.fromisoformat(val) if val else None
 
 
+def _subscription_rows(rows) -> list[dict]:
+    return [{"project_id": r.project_id, "rulebook_id": r.rulebook_id} for r in rows]
+
+
+def _rule_suppression_rows(rows) -> list[dict]:
+    return [{"project_id": r.project_id, "rule_id": r.rule_id} for r in rows]
+
+
+def _topic_suppression_rows(rows) -> list[dict]:
+    return [{"project_id": r.project_id, "topic_id": r.topic_id} for r in rows]
+
+
 # ---------------------------------------------------------------------------
 # Export
 # ---------------------------------------------------------------------------
 
 async def export_full_backup() -> dict:
-    """Export all data as version-2 JSON backup."""
+    """Export all data as a version-3 JSON backup."""
     async with async_session() as session:
         users = (await session.execute(select(User))).scalars().all()
         projects = (await session.execute(select(Project))).scalars().all()
@@ -41,15 +77,29 @@ async def export_full_backup() -> dict:
             select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id)
         )).scalars().all()
         settings = (await session.execute(select(Setting))).scalars().all()
+        rulebooks = (await session.execute(select(Rulebook))).scalars().all()
+        topics = (await session.execute(select(RulebookTopic))).scalars().all()
+        rules = (await session.execute(select(Rule))).scalars().all()
+        events = (await session.execute(select(Event))).scalars().all()
+        subscriptions = (await session.execute(
+            select(project_rulebook_subscriptions)
+        )).all()
+        rule_suppressions = (await session.execute(
+            select(project_rule_suppressions)
+        )).all()
+        topic_suppressions = (await session.execute(
+            select(project_topic_suppressions)
+        )).all()
 
     return {
-        "version": 2,
+        "version": BACKUP_VERSION,
         "scope": "full",
         "exported_at": datetime.now(timezone.utc).isoformat(),
         "_security_notice": (
             "This backup contains hashed passwords. "
             "Store it securely and restrict access."
         ),
+        "_not_included": _NOT_INCLUDED,
         "users": [
             {
                 "id": u.id,
@@ -153,16 +203,80 @@ async def export_full_backup() -> dict:
             {"user_id": s.user_id, "key": s.key, "value": s.value}
             for s in settings
         ],
+        "rulebooks": [
+            {
+                "id": rb.id,
+                "owner_user_id": rb.owner_user_id,
+                "title": rb.title,
+                "description": rb.description,
+                "always_on": rb.always_on,
+                "created_at": rb.created_at.isoformat(),
+                "updated_at": rb.updated_at.isoformat(),
+            }
+            for rb in rulebooks
+        ],
+        "rulebook_topics": [
+            {
+                "id": t.id,
+                "rulebook_id": t.rulebook_id,
+                "title": t.title,
+                "description": t.description,
+                "order_index": t.order_index,
+                "created_at": t.created_at.isoformat(),
+                "updated_at": t.updated_at.isoformat(),
+            }
+            for t in topics
+        ],
+        "rules": [
+            {
+                "id": r.id,
+                "topic_id": r.topic_id,
+                "project_id": r.project_id,
+                "title": r.title,
+                "statement": r.statement,
+                "why": r.why,
+                "how_to_apply": r.how_to_apply,
+                "order_index": r.order_index,
+                "created_at": r.created_at.isoformat(),
+                "updated_at": r.updated_at.isoformat(),
+            }
+            for r in rules
+        ],
+        "rulebook_subscriptions": _subscription_rows(subscriptions),
+        "rule_suppressions": _rule_suppression_rows(rule_suppressions),
+        "topic_suppressions": _topic_suppression_rows(topic_suppressions),
+        "events": [
+            {
+                "id": e.id,
+                "user_id": e.user_id,
+                "project_id": e.project_id,
+                "uid": e.uid,
+                "caldav_uid": e.caldav_uid,
+                "title": e.title,
+                "start_dt": e.start_dt.isoformat() if e.start_dt else None,
+                "duration_minutes": e.duration_minutes,
+                "all_day": e.all_day,
+                "description": e.description,
+                "location": e.location,
+                "color": e.color,
+                "recurrence": e.recurrence,
+                "reminder_minutes": e.reminder_minutes,
+                "created_at": e.created_at.isoformat(),
+                "updated_at": e.updated_at.isoformat(),
+            }
+            for e in events
+        ],
     }
 
 
 async def export_user_backup(user_id: int) -> dict:
-    """Export a single user's data as version-2 JSON backup."""
+    """Export a single user's data as a version-3 JSON backup."""
     async with async_session() as session:
         user = await session.get(User, user_id)
         projects = (await session.execute(
             select(Project).where(Project.user_id == user_id)
         )).scalars().all()
+        project_ids = [p.id for p in projects]
         milestones = (await session.execute(
             select(Milestone).where(Milestone.user_id == user_id)
         )).scalars().all()
@@ -182,11 +296,51 @@ async def export_user_backup(user_id: int) -> dict:
         settings = (await session.execute(
             select(Setting).where(Setting.user_id == user_id)
         )).scalars().all()
+        rulebooks = (await session.execute(
+            select(Rulebook).where(Rulebook.owner_user_id == user_id)
+        )).scalars().all()
+        rulebook_ids = [rb.id for rb in rulebooks]
+        topics = (await session.execute(
+            select(RulebookTopic).where(RulebookTopic.rulebook_id.in_(rulebook_ids))
+        )).scalars().all() if rulebook_ids else []
+        topic_ids = [t.id for t in topics]
+        # Rules owned by the user = rules in the user's topics OR project-rules
+        # on the user's projects.
+        rule_filters = []
+        if topic_ids:
+            rule_filters.append(Rule.topic_id.in_(topic_ids))
+        if project_ids:
+            rule_filters.append(Rule.project_id.in_(project_ids))
+        rules = (await session.execute(
+            select(Rule).where(or_(*rule_filters))
+        )).scalars().all() if rule_filters else []
+        events = (await session.execute(
+            select(Event).where(Event.user_id == user_id)
+        )).scalars().all()
+        if project_ids:
+            subscriptions = (await session.execute(
+                select(project_rulebook_subscriptions).where(
+                    project_rulebook_subscriptions.c.project_id.in_(project_ids)
+                )
+            )).all()
+            rule_suppressions = (await session.execute(
+                select(project_rule_suppressions).where(
+                    project_rule_suppressions.c.project_id.in_(project_ids)
+                )
+            )).all()
+            topic_suppressions = (await session.execute(
+                select(project_topic_suppressions).where(
+                    project_topic_suppressions.c.project_id.in_(project_ids)
+                )
+            )).all()
+        else:
+            subscriptions = rule_suppressions = topic_suppressions = []
 
     return {
-        "version": 2,
+        "version": BACKUP_VERSION,
         "scope": "user",
         "exported_at": datetime.now(timezone.utc).isoformat(),
+        "_not_included": _NOT_INCLUDED,
         "user": {
             "id": user.id,
             "username": user.username,
@@ -225,6 +379,7 @@ async def export_user_backup(user_id: int) -> dict:
         "notes": [
             {
                 "id": n.id,
+                "user_id": n.user_id,
                 "title": n.title,
                 "body": n.body,
                 "tags": n.tags or [],
@@ -242,6 +397,7 @@ async def export_user_backup(user_id: int) -> dict:
         "task_logs": [
             {
                 "id": tl.id,
+                "user_id": tl.user_id,
                 "task_id": tl.task_id,
                 "content": tl.content,
                 "duration_minutes": tl.duration_minutes,
@@ -253,6 +409,7 @@ async def export_user_backup(user_id: int) -> dict:
         "note_drafts": [
             {
                 "id": nd.id,
+                "user_id": nd.user_id,
                 "note_id": nd.note_id,
                 "proposed_body": nd.proposed_body,
                 "original_body": nd.original_body,
@@ -266,6 +423,7 @@ async def export_user_backup(user_id: int) -> dict:
         "note_versions": [
             {
                 "id": nv.id,
+                "user_id": nv.user_id,
                 "note_id": nv.note_id,
                 "title": nv.title,
                 "body": nv.body,
@@ -277,9 +435,72 @@ async def export_user_backup(user_id: int) -> dict:
             for nv in note_versions
         ],
         "settings": [
-            {"key": s.key, "value": s.value}
+            {"user_id": s.user_id, "key": s.key, "value": s.value}
             for s in settings
         ],
+        "rulebooks": [
+            {
+                "id": rb.id,
+                "owner_user_id": rb.owner_user_id,
+                "title": rb.title,
+                "description": rb.description,
+                "always_on": rb.always_on,
+                "created_at": rb.created_at.isoformat(),
+                "updated_at": rb.updated_at.isoformat(),
+            }
+            for rb in rulebooks
+        ],
+        "rulebook_topics": [
+            {
+                "id": t.id,
+                "rulebook_id": t.rulebook_id,
+                "title": t.title,
+                "description": t.description,
+                "order_index": t.order_index,
+                "created_at": t.created_at.isoformat(),
+                "updated_at": t.updated_at.isoformat(),
+            }
+            for t in topics
+        ],
+        "rules": [
+            {
+                "id": r.id,
+                "topic_id": r.topic_id,
+                "project_id": r.project_id,
+                "title": r.title,
+                "statement": r.statement,
+                "why": r.why,
+                "how_to_apply": r.how_to_apply,
+                "order_index": r.order_index,
+                "created_at": r.created_at.isoformat(),
+                "updated_at": r.updated_at.isoformat(),
+            }
+            for r in rules
+        ],
+        "rulebook_subscriptions": _subscription_rows(subscriptions),
+        "rule_suppressions": _rule_suppression_rows(rule_suppressions),
+        "topic_suppressions": _topic_suppression_rows(topic_suppressions),
+        "events": [
+            {
+                "id": e.id,
+                "user_id": e.user_id,
+                "project_id": e.project_id,
+                "uid": e.uid,
+                "caldav_uid": e.caldav_uid,
+                "title": e.title,
+                "start_dt": e.start_dt.isoformat() if e.start_dt else None,
+                "duration_minutes": e.duration_minutes,
+                "all_day": e.all_day,
+                "description": e.description,
+                "location": e.location,
+                "color": e.color,
+                "recurrence": e.recurrence,
+                "reminder_minutes": e.reminder_minutes,
+                "created_at": e.created_at.isoformat(),
+                "updated_at": e.updated_at.isoformat(),
+            }
+            for e in events
+        ],
     }
 
 
@@ -292,6 +513,8 @@ async def restore_full_backup(data: dict) -> dict:
     version = data.get("version", 1)
     if version == 1:
         return await _restore_v1(data)
+    # v2 and v3 share one path; v3-only sections are guarded by data.get so a
+    # v2 payload (without them) restores cleanly.
     return await _restore_v2(data)
 
 
@@ -366,15 +589,19 @@ async def _restore_v1(data: dict) -> dict:
 
 
 async def _restore_v2(data: dict) -> dict:
-    """Restore v2 backup with full FK re-mapping.
+    """Restore v2/v3 backup with full FK re-mapping.
 
     Conversations + push subscriptions in pre-pivot backups are silently
-    skipped — those subsystems were removed in the MCP-first pivot.
+    skipped — those subsystems were removed in the MCP-first pivot. v3-only
+    sections (rulebooks/topics/rules/join-tables/events) are guarded by
+    data.get so a v2 payload restores without them.
     """
     stats: dict[str, int] = {
         "users": 0, "projects": 0, "milestones": 0, "notes": 0,
         "task_logs": 0, "note_drafts": 0, "note_versions": 0,
-        "settings": 0,
+        "settings": 0, "rulebooks": 0, "rulebook_topics": 0, "rules": 0,
+        "rulebook_subscriptions": 0, "rule_suppressions": 0,
+        "topic_suppressions": 0, "events": 0,
     }
 
     async with async_session() as session:
@@ -382,6 +609,9 @@ async def _restore_v2(data: dict) -> dict:
         project_id_map: dict[int, int] = {}
         milestone_id_map: dict[int, int] = {}
         note_id_map: dict[int, int] = {}
+        rulebook_id_map: dict[int, int] = {}
+        topic_id_map: dict[int, int] = {}
+        rule_id_map: dict[int, int] = {}
 
         # 1. Users
         for u_data in data.get("users", []):
@@ -539,7 +769,123 @@ async def _restore_v2(data: dict) -> dict:
             session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
             stats["settings"] += 1
 
+        # 9. Rulebooks (v3)
+        for rb_data in data.get("rulebooks", []):
+            mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
+            if mapped_uid is None:
+                continue
+            rb = Rulebook(
+                owner_user_id=mapped_uid,
+                title=rb_data.get("title", ""),
+                description=rb_data.get("description", ""),
+                always_on=rb_data.get("always_on", False),
+                created_at=_dt(rb_data.get("created_at")),
+                updated_at=_dt(rb_data.get("updated_at")),
+            )
+            session.add(rb)
+            await session.flush()
+            rulebook_id_map[rb_data["id"]] = rb.id
+            stats["rulebooks"] += 1
+
+        # 10. Topics (v3)
+        for t_data in data.get("rulebook_topics", []):
+            mapped_rbid = rulebook_id_map.get(t_data.get("rulebook_id", 0))
+            if mapped_rbid is None:
+                continue
+            topic = RulebookTopic(
+                rulebook_id=mapped_rbid,
+                title=t_data.get("title", ""),
+                description=t_data.get("description"),
+                order_index=t_data.get("order_index", 0),
+                created_at=_dt(t_data.get("created_at")),
+                updated_at=_dt(t_data.get("updated_at")),
+            )
+            session.add(topic)
+            await session.flush()
+            topic_id_map[t_data["id"]] = topic.id
+            stats["rulebook_topics"] += 1
+
+        # 11. Rules (v3) — topic-rule (topic_id) XOR project-rule (project_id)
+        for r_data in data.get("rules", []):
+            mapped_topic = topic_id_map.get(r_data["topic_id"]) if r_data.get("topic_id") else None
+            mapped_proj = project_id_map.get(r_data["project_id"]) if r_data.get("project_id") else None
+            if mapped_topic is None and mapped_proj is None:
+                continue  # orphaned — its parent didn't restore
+            rule = Rule(
+                topic_id=mapped_topic,
+                project_id=mapped_proj,
+                title=r_data.get("title", ""),
+                statement=r_data.get("statement", ""),
+                why=r_data.get("why") or None,
+                how_to_apply=r_data.get("how_to_apply") or None,
+                order_index=r_data.get("order_index", 0),
+                created_at=_dt(r_data.get("created_at")),
+                updated_at=_dt(r_data.get("updated_at")),
+            )
+            session.add(rule)
+            await session.flush()
+            rule_id_map[r_data["id"]] = rule.id
+            stats["rules"] += 1
+
+        # 12. Rulebook subscriptions (v3 join table)
+        for sub in data.get("rulebook_subscriptions", []):
+            mapped_pid = project_id_map.get(sub.get("project_id", 0))
+            mapped_rbid = rulebook_id_map.get(sub.get("rulebook_id", 0))
+            if mapped_pid is None or mapped_rbid is None:
+                continue
+            await session.execute(project_rulebook_subscriptions.insert().values(
+                project_id=mapped_pid, rulebook_id=mapped_rbid,
+            ))
+            stats["rulebook_subscriptions"] += 1
+
+        # 13. Rule suppressions (v3 join table)
+        for sup in data.get("rule_suppressions", []):
+            mapped_pid = project_id_map.get(sup.get("project_id", 0))
+            mapped_rid = rule_id_map.get(sup.get("rule_id", 0))
+            if mapped_pid is None or mapped_rid is None:
+                continue
+            await session.execute(project_rule_suppressions.insert().values(
+                project_id=mapped_pid, rule_id=mapped_rid,
+            ))
+            stats["rule_suppressions"] += 1
+
+        # 14. Topic suppressions (v3 join table)
+        for sup in data.get("topic_suppressions", []):
+            mapped_pid = project_id_map.get(sup.get("project_id", 0))
+            mapped_tid = topic_id_map.get(sup.get("topic_id", 0))
+            if mapped_pid is None or mapped_tid is None:
+                continue
+            await session.execute(project_topic_suppressions.insert().values(
+                project_id=mapped_pid, topic_id=mapped_tid,
+            ))
+            stats["topic_suppressions"] += 1
+
+        # 15. Events (v3)
+        for e_data in data.get("events", []):
+            mapped_uid = user_id_map.get(e_data.get("user_id", 0))
+            if mapped_uid is None:
+                continue
+            ev = Event(
+                user_id=mapped_uid,
+                project_id=project_id_map.get(e_data["project_id"]) if e_data.get("project_id") else None,
+                uid=e_data.get("uid", ""),
+                caldav_uid=e_data.get("caldav_uid", ""),
+                title=e_data.get("title", ""),
+                start_dt=_dt(e_data.get("start_dt")),
+                duration_minutes=e_data.get("duration_minutes"),
+                all_day=e_data.get("all_day", False),
+                description=e_data.get("description", ""),
+                location=e_data.get("location", ""),
+                color=e_data.get("color", ""),
+                recurrence=e_data.get("recurrence"),
+                reminder_minutes=e_data.get("reminder_minutes"),
+                created_at=_dt(e_data.get("created_at")),
+                updated_at=_dt(e_data.get("updated_at")),
+            )
+            session.add(ev)
+            stats["events"] += 1
+
         await session.commit()
 
-    logger.info("Restored v2 backup: %s", stats)
+    logger.info("Restored v2/v3 backup: %s", stats)
     return stats

tests/test_services_backup.py

@@ -0,0 +1,72 @@
+"""Unit tests for the v3 backup export contract.
+
+CI runs pytest with no database, so these cover the parts that don't need one:
+the version/coverage constants, the pure join-table row helpers, and the export
+dict shape (via a mocked session). Full FK-remapping round-trip is exercised
+manually against a real DB (export a backup, confirm rulebooks/events appear).
+"""
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from scribe.services import backup
+
+
+def test_backup_version_is_v3():
+    assert backup.BACKUP_VERSION == 3
+
+
+def test_not_included_lists_the_known_gaps():
+    # The deferred tables must be surfaced explicitly, not silently dropped.
+    for table in ("groups", "project_shares", "note_shares", "api_keys", "embeddings"):
+        assert table in backup._NOT_INCLUDED
+
+
+def test_join_table_row_helpers_are_pure():
+    subs = [SimpleNamespace(project_id=1, rulebook_id=2)]
+    rsup = [SimpleNamespace(project_id=1, rule_id=9)]
+    tsup = [SimpleNamespace(project_id=1, topic_id=7)]
+    assert backup._subscription_rows(subs) == [{"project_id": 1, "rulebook_id": 2}]
+    assert backup._rule_suppression_rows(rsup) == [{"project_id": 1, "rule_id": 9}]
+    assert backup._topic_suppression_rows(tsup) == [{"project_id": 1, "topic_id": 7}]
+
+
+class _Result:
+    def scalars(self):
+        return self
+
+    def all(self):
+        return []
+
+
+class _Session:
+    async def execute(self, *a, **k):
+        return _Result()
+
+    async def get(self, *a, **k):
+        return None
+
+
+class _CM:
+    async def __aenter__(self):
+        return _Session()
+
+    async def __aexit__(self, *a):
+        return False
+
+
+@pytest.mark.asyncio
+async def test_export_full_backup_contains_v3_sections():
+    with patch("scribe.services.backup.async_session", lambda: _CM()):
+        out = await backup.export_full_backup()
+
+    assert out["version"] == 3
+    assert out["scope"] == "full"
+    assert "api_keys" in out["_not_included"]
+    # The sections v2 silently dropped must now be present (empty here).
+    for key in ("rulebooks", "rulebook_topics", "rules",
+                "rulebook_subscriptions", "rule_suppressions",
+                "topic_suppressions", "events"):
+        assert key in out, f"missing v3 section: {key}"
+        assert out[key] == []