diff --git a/src/scribe/services/backup.py b/src/scribe/services/backup.py index 7d1095e..9184c12 100644 --- a/src/scribe/services/backup.py +++ b/src/scribe/services/backup.py @@ -1,20 +1,44 @@ import logging from datetime import date, datetime, timezone -from sqlalchemy import select +from sqlalchemy import or_, select from scribe.models import async_session +from scribe.models.event import Event from scribe.models.milestone import Milestone from scribe.models.note import Note from scribe.models.note_draft import NoteDraft from scribe.models.note_version import NoteVersion from scribe.models.project import Project +from scribe.models.rulebook import ( + Rule, + Rulebook, + RulebookTopic, + project_rule_suppressions, + project_rulebook_subscriptions, + project_topic_suppressions, +) from scribe.models.setting import Setting from scribe.models.task_log import TaskLog from scribe.models.user import User logger = logging.getLogger(__name__) +# Backup format version. v3 (2026-06) added rulebooks/topics/rules + their +# project subscription/suppression join tables, and events — all silently +# dropped by v2. Bump when the serialized schema changes. +BACKUP_VERSION = 3 + +# Tables intentionally NOT in the backup, surfaced in the payload so the gap is +# explicit rather than silent. ACL (groups/shares) is a coherent follow-up; +# embeddings are derived (regenerated from note bodies); api_keys are sensitive +# credentials; the rest are transient/operational. +_NOT_INCLUDED = [ + "groups", "group_memberships", "project_shares", "note_shares", + "api_keys", "embeddings", "app_logs", "notifications", "invitations", + "password_resets", "user_profiles", +] + def _dt(val: str | None) -> datetime: return datetime.fromisoformat(val) if val else datetime.now(timezone.utc) @@ -24,12 +48,24 @@ def _d(val: str | None) -> date | None: return date.fromisoformat(val) if val else None +def _subscription_rows(rows) -> list[dict]: + return [{"project_id": r.project_id, "rulebook_id": r.rulebook_id} for r in rows] + + +def _rule_suppression_rows(rows) -> list[dict]: + return [{"project_id": r.project_id, "rule_id": r.rule_id} for r in rows] + + +def _topic_suppression_rows(rows) -> list[dict]: + return [{"project_id": r.project_id, "topic_id": r.topic_id} for r in rows] + + # --------------------------------------------------------------------------- # Export # --------------------------------------------------------------------------- async def export_full_backup() -> dict: - """Export all data as version-2 JSON backup.""" + """Export all data as a version-3 JSON backup.""" async with async_session() as session: users = (await session.execute(select(User))).scalars().all() projects = (await session.execute(select(Project))).scalars().all() @@ -41,15 +77,29 @@ async def export_full_backup() -> dict: select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id) )).scalars().all() settings = (await session.execute(select(Setting))).scalars().all() + rulebooks = (await session.execute(select(Rulebook))).scalars().all() + topics = (await session.execute(select(RulebookTopic))).scalars().all() + rules = (await session.execute(select(Rule))).scalars().all() + events = (await session.execute(select(Event))).scalars().all() + subscriptions = (await session.execute( + select(project_rulebook_subscriptions) + )).all() + rule_suppressions = (await session.execute( + select(project_rule_suppressions) + )).all() + topic_suppressions = (await session.execute( + select(project_topic_suppressions) + )).all() return { - "version": 2, + "version": BACKUP_VERSION, "scope": "full", "exported_at": datetime.now(timezone.utc).isoformat(), "_security_notice": ( "This backup contains hashed passwords. " "Store it securely and restrict access." ), + "_not_included": _NOT_INCLUDED, "users": [ { "id": u.id, @@ -153,16 +203,80 @@ async def export_full_backup() -> dict: {"user_id": s.user_id, "key": s.key, "value": s.value} for s in settings ], + "rulebooks": [ + { + "id": rb.id, + "owner_user_id": rb.owner_user_id, + "title": rb.title, + "description": rb.description, + "always_on": rb.always_on, + "created_at": rb.created_at.isoformat(), + "updated_at": rb.updated_at.isoformat(), + } + for rb in rulebooks + ], + "rulebook_topics": [ + { + "id": t.id, + "rulebook_id": t.rulebook_id, + "title": t.title, + "description": t.description, + "order_index": t.order_index, + "created_at": t.created_at.isoformat(), + "updated_at": t.updated_at.isoformat(), + } + for t in topics + ], + "rules": [ + { + "id": r.id, + "topic_id": r.topic_id, + "project_id": r.project_id, + "title": r.title, + "statement": r.statement, + "why": r.why, + "how_to_apply": r.how_to_apply, + "order_index": r.order_index, + "created_at": r.created_at.isoformat(), + "updated_at": r.updated_at.isoformat(), + } + for r in rules + ], + "rulebook_subscriptions": _subscription_rows(subscriptions), + "rule_suppressions": _rule_suppression_rows(rule_suppressions), + "topic_suppressions": _topic_suppression_rows(topic_suppressions), + "events": [ + { + "id": e.id, + "user_id": e.user_id, + "project_id": e.project_id, + "uid": e.uid, + "caldav_uid": e.caldav_uid, + "title": e.title, + "start_dt": e.start_dt.isoformat() if e.start_dt else None, + "duration_minutes": e.duration_minutes, + "all_day": e.all_day, + "description": e.description, + "location": e.location, + "color": e.color, + "recurrence": e.recurrence, + "reminder_minutes": e.reminder_minutes, + "created_at": e.created_at.isoformat(), + "updated_at": e.updated_at.isoformat(), + } + for e in events + ], } async def export_user_backup(user_id: int) -> dict: - """Export a single user's data as version-2 JSON backup.""" + """Export a single user's data as a version-3 JSON backup.""" async with async_session() as session: user = await session.get(User, user_id) projects = (await session.execute( select(Project).where(Project.user_id == user_id) )).scalars().all() + project_ids = [p.id for p in projects] milestones = (await session.execute( select(Milestone).where(Milestone.user_id == user_id) )).scalars().all() @@ -182,11 +296,51 @@ async def export_user_backup(user_id: int) -> dict: settings = (await session.execute( select(Setting).where(Setting.user_id == user_id) )).scalars().all() + rulebooks = (await session.execute( + select(Rulebook).where(Rulebook.owner_user_id == user_id) + )).scalars().all() + rulebook_ids = [rb.id for rb in rulebooks] + topics = (await session.execute( + select(RulebookTopic).where(RulebookTopic.rulebook_id.in_(rulebook_ids)) + )).scalars().all() if rulebook_ids else [] + topic_ids = [t.id for t in topics] + # Rules owned by the user = rules in the user's topics OR project-rules + # on the user's projects. + rule_filters = [] + if topic_ids: + rule_filters.append(Rule.topic_id.in_(topic_ids)) + if project_ids: + rule_filters.append(Rule.project_id.in_(project_ids)) + rules = (await session.execute( + select(Rule).where(or_(*rule_filters)) + )).scalars().all() if rule_filters else [] + events = (await session.execute( + select(Event).where(Event.user_id == user_id) + )).scalars().all() + if project_ids: + subscriptions = (await session.execute( + select(project_rulebook_subscriptions).where( + project_rulebook_subscriptions.c.project_id.in_(project_ids) + ) + )).all() + rule_suppressions = (await session.execute( + select(project_rule_suppressions).where( + project_rule_suppressions.c.project_id.in_(project_ids) + ) + )).all() + topic_suppressions = (await session.execute( + select(project_topic_suppressions).where( + project_topic_suppressions.c.project_id.in_(project_ids) + ) + )).all() + else: + subscriptions = rule_suppressions = topic_suppressions = [] return { - "version": 2, + "version": BACKUP_VERSION, "scope": "user", "exported_at": datetime.now(timezone.utc).isoformat(), + "_not_included": _NOT_INCLUDED, "user": { "id": user.id, "username": user.username, @@ -225,6 +379,7 @@ async def export_user_backup(user_id: int) -> dict: "notes": [ { "id": n.id, + "user_id": n.user_id, "title": n.title, "body": n.body, "tags": n.tags or [], @@ -242,6 +397,7 @@ async def export_user_backup(user_id: int) -> dict: "task_logs": [ { "id": tl.id, + "user_id": tl.user_id, "task_id": tl.task_id, "content": tl.content, "duration_minutes": tl.duration_minutes, @@ -253,6 +409,7 @@ async def export_user_backup(user_id: int) -> dict: "note_drafts": [ { "id": nd.id, + "user_id": nd.user_id, "note_id": nd.note_id, "proposed_body": nd.proposed_body, "original_body": nd.original_body, @@ -266,6 +423,7 @@ async def export_user_backup(user_id: int) -> dict: "note_versions": [ { "id": nv.id, + "user_id": nv.user_id, "note_id": nv.note_id, "title": nv.title, "body": nv.body, @@ -277,9 +435,72 @@ async def export_user_backup(user_id: int) -> dict: for nv in note_versions ], "settings": [ - {"key": s.key, "value": s.value} + {"user_id": s.user_id, "key": s.key, "value": s.value} for s in settings ], + "rulebooks": [ + { + "id": rb.id, + "owner_user_id": rb.owner_user_id, + "title": rb.title, + "description": rb.description, + "always_on": rb.always_on, + "created_at": rb.created_at.isoformat(), + "updated_at": rb.updated_at.isoformat(), + } + for rb in rulebooks + ], + "rulebook_topics": [ + { + "id": t.id, + "rulebook_id": t.rulebook_id, + "title": t.title, + "description": t.description, + "order_index": t.order_index, + "created_at": t.created_at.isoformat(), + "updated_at": t.updated_at.isoformat(), + } + for t in topics + ], + "rules": [ + { + "id": r.id, + "topic_id": r.topic_id, + "project_id": r.project_id, + "title": r.title, + "statement": r.statement, + "why": r.why, + "how_to_apply": r.how_to_apply, + "order_index": r.order_index, + "created_at": r.created_at.isoformat(), + "updated_at": r.updated_at.isoformat(), + } + for r in rules + ], + "rulebook_subscriptions": _subscription_rows(subscriptions), + "rule_suppressions": _rule_suppression_rows(rule_suppressions), + "topic_suppressions": _topic_suppression_rows(topic_suppressions), + "events": [ + { + "id": e.id, + "user_id": e.user_id, + "project_id": e.project_id, + "uid": e.uid, + "caldav_uid": e.caldav_uid, + "title": e.title, + "start_dt": e.start_dt.isoformat() if e.start_dt else None, + "duration_minutes": e.duration_minutes, + "all_day": e.all_day, + "description": e.description, + "location": e.location, + "color": e.color, + "recurrence": e.recurrence, + "reminder_minutes": e.reminder_minutes, + "created_at": e.created_at.isoformat(), + "updated_at": e.updated_at.isoformat(), + } + for e in events + ], } @@ -292,6 +513,8 @@ async def restore_full_backup(data: dict) -> dict: version = data.get("version", 1) if version == 1: return await _restore_v1(data) + # v2 and v3 share one path; v3-only sections are guarded by data.get so a + # v2 payload (without them) restores cleanly. return await _restore_v2(data) @@ -366,15 +589,19 @@ async def _restore_v1(data: dict) -> dict: async def _restore_v2(data: dict) -> dict: - """Restore v2 backup with full FK re-mapping. + """Restore v2/v3 backup with full FK re-mapping. Conversations + push subscriptions in pre-pivot backups are silently - skipped — those subsystems were removed in the MCP-first pivot. + skipped — those subsystems were removed in the MCP-first pivot. v3-only + sections (rulebooks/topics/rules/join-tables/events) are guarded by + data.get so a v2 payload restores without them. """ stats: dict[str, int] = { "users": 0, "projects": 0, "milestones": 0, "notes": 0, "task_logs": 0, "note_drafts": 0, "note_versions": 0, - "settings": 0, + "settings": 0, "rulebooks": 0, "rulebook_topics": 0, "rules": 0, + "rulebook_subscriptions": 0, "rule_suppressions": 0, + "topic_suppressions": 0, "events": 0, } async with async_session() as session: @@ -382,6 +609,9 @@ async def _restore_v2(data: dict) -> dict: project_id_map: dict[int, int] = {} milestone_id_map: dict[int, int] = {} note_id_map: dict[int, int] = {} + rulebook_id_map: dict[int, int] = {} + topic_id_map: dict[int, int] = {} + rule_id_map: dict[int, int] = {} # 1. Users for u_data in data.get("users", []): @@ -539,7 +769,123 @@ async def _restore_v2(data: dict) -> dict: session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", ""))) stats["settings"] += 1 + # 9. Rulebooks (v3) + for rb_data in data.get("rulebooks", []): + mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0)) + if mapped_uid is None: + continue + rb = Rulebook( + owner_user_id=mapped_uid, + title=rb_data.get("title", ""), + description=rb_data.get("description", ""), + always_on=rb_data.get("always_on", False), + created_at=_dt(rb_data.get("created_at")), + updated_at=_dt(rb_data.get("updated_at")), + ) + session.add(rb) + await session.flush() + rulebook_id_map[rb_data["id"]] = rb.id + stats["rulebooks"] += 1 + + # 10. Topics (v3) + for t_data in data.get("rulebook_topics", []): + mapped_rbid = rulebook_id_map.get(t_data.get("rulebook_id", 0)) + if mapped_rbid is None: + continue + topic = RulebookTopic( + rulebook_id=mapped_rbid, + title=t_data.get("title", ""), + description=t_data.get("description"), + order_index=t_data.get("order_index", 0), + created_at=_dt(t_data.get("created_at")), + updated_at=_dt(t_data.get("updated_at")), + ) + session.add(topic) + await session.flush() + topic_id_map[t_data["id"]] = topic.id + stats["rulebook_topics"] += 1 + + # 11. Rules (v3) — topic-rule (topic_id) XOR project-rule (project_id) + for r_data in data.get("rules", []): + mapped_topic = topic_id_map.get(r_data["topic_id"]) if r_data.get("topic_id") else None + mapped_proj = project_id_map.get(r_data["project_id"]) if r_data.get("project_id") else None + if mapped_topic is None and mapped_proj is None: + continue # orphaned — its parent didn't restore + rule = Rule( + topic_id=mapped_topic, + project_id=mapped_proj, + title=r_data.get("title", ""), + statement=r_data.get("statement", ""), + why=r_data.get("why") or None, + how_to_apply=r_data.get("how_to_apply") or None, + order_index=r_data.get("order_index", 0), + created_at=_dt(r_data.get("created_at")), + updated_at=_dt(r_data.get("updated_at")), + ) + session.add(rule) + await session.flush() + rule_id_map[r_data["id"]] = rule.id + stats["rules"] += 1 + + # 12. Rulebook subscriptions (v3 join table) + for sub in data.get("rulebook_subscriptions", []): + mapped_pid = project_id_map.get(sub.get("project_id", 0)) + mapped_rbid = rulebook_id_map.get(sub.get("rulebook_id", 0)) + if mapped_pid is None or mapped_rbid is None: + continue + await session.execute(project_rulebook_subscriptions.insert().values( + project_id=mapped_pid, rulebook_id=mapped_rbid, + )) + stats["rulebook_subscriptions"] += 1 + + # 13. Rule suppressions (v3 join table) + for sup in data.get("rule_suppressions", []): + mapped_pid = project_id_map.get(sup.get("project_id", 0)) + mapped_rid = rule_id_map.get(sup.get("rule_id", 0)) + if mapped_pid is None or mapped_rid is None: + continue + await session.execute(project_rule_suppressions.insert().values( + project_id=mapped_pid, rule_id=mapped_rid, + )) + stats["rule_suppressions"] += 1 + + # 14. Topic suppressions (v3 join table) + for sup in data.get("topic_suppressions", []): + mapped_pid = project_id_map.get(sup.get("project_id", 0)) + mapped_tid = topic_id_map.get(sup.get("topic_id", 0)) + if mapped_pid is None or mapped_tid is None: + continue + await session.execute(project_topic_suppressions.insert().values( + project_id=mapped_pid, topic_id=mapped_tid, + )) + stats["topic_suppressions"] += 1 + + # 15. Events (v3) + for e_data in data.get("events", []): + mapped_uid = user_id_map.get(e_data.get("user_id", 0)) + if mapped_uid is None: + continue + ev = Event( + user_id=mapped_uid, + project_id=project_id_map.get(e_data["project_id"]) if e_data.get("project_id") else None, + uid=e_data.get("uid", ""), + caldav_uid=e_data.get("caldav_uid", ""), + title=e_data.get("title", ""), + start_dt=_dt(e_data.get("start_dt")), + duration_minutes=e_data.get("duration_minutes"), + all_day=e_data.get("all_day", False), + description=e_data.get("description", ""), + location=e_data.get("location", ""), + color=e_data.get("color", ""), + recurrence=e_data.get("recurrence"), + reminder_minutes=e_data.get("reminder_minutes"), + created_at=_dt(e_data.get("created_at")), + updated_at=_dt(e_data.get("updated_at")), + ) + session.add(ev) + stats["events"] += 1 + await session.commit() - logger.info("Restored v2 backup: %s", stats) + logger.info("Restored v2/v3 backup: %s", stats) return stats diff --git a/tests/test_services_backup.py b/tests/test_services_backup.py new file mode 100644 index 0000000..babe2e0 --- /dev/null +++ b/tests/test_services_backup.py @@ -0,0 +1,72 @@ +"""Unit tests for the v3 backup export contract. + +CI runs pytest with no database, so these cover the parts that don't need one: +the version/coverage constants, the pure join-table row helpers, and the export +dict shape (via a mocked session). Full FK-remapping round-trip is exercised +manually against a real DB (export a backup, confirm rulebooks/events appear). +""" +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from scribe.services import backup + + +def test_backup_version_is_v3(): + assert backup.BACKUP_VERSION == 3 + + +def test_not_included_lists_the_known_gaps(): + # The deferred tables must be surfaced explicitly, not silently dropped. + for table in ("groups", "project_shares", "note_shares", "api_keys", "embeddings"): + assert table in backup._NOT_INCLUDED + + +def test_join_table_row_helpers_are_pure(): + subs = [SimpleNamespace(project_id=1, rulebook_id=2)] + rsup = [SimpleNamespace(project_id=1, rule_id=9)] + tsup = [SimpleNamespace(project_id=1, topic_id=7)] + assert backup._subscription_rows(subs) == [{"project_id": 1, "rulebook_id": 2}] + assert backup._rule_suppression_rows(rsup) == [{"project_id": 1, "rule_id": 9}] + assert backup._topic_suppression_rows(tsup) == [{"project_id": 1, "topic_id": 7}] + + +class _Result: + def scalars(self): + return self + + def all(self): + return [] + + +class _Session: + async def execute(self, *a, **k): + return _Result() + + async def get(self, *a, **k): + return None + + +class _CM: + async def __aenter__(self): + return _Session() + + async def __aexit__(self, *a): + return False + + +@pytest.mark.asyncio +async def test_export_full_backup_contains_v3_sections(): + with patch("scribe.services.backup.async_session", lambda: _CM()): + out = await backup.export_full_backup() + + assert out["version"] == 3 + assert out["scope"] == "full" + assert "api_keys" in out["_not_included"] + # The sections v2 silently dropped must now be present (empty here). + for key in ("rulebooks", "rulebook_topics", "rules", + "rulebook_subscriptions", "rule_suppressions", + "topic_suppressions", "events"): + assert key in out, f"missing v3 section: {key}" + assert out[key] == []