From 878b149f4d0a20588913191a4dc225661f761eab Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 11 Mar 2026 11:57:21 -0400 Subject: [PATCH] feat(backup): v2 backup/restore with all models and FK re-mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export now includes: projects, milestones, task_logs, note_drafts, note_versions, push_subscriptions; full user fields (oauth_sub, session_version); full message fields (status, tool_calls). Restore v2 builds ID maps for all tables and patches FKs in dependency order (users → projects → milestones → notes → child records). Notes restored in two passes to handle self-referential parent_id correctly. v1 backups still restore via _restore_v1 path (parent_id bug also fixed). Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/services/backup.py | 717 +++++++++++++++++++------ 1 file changed, 555 insertions(+), 162 deletions(-) diff --git a/src/fabledassistant/services/backup.py b/src/fabledassistant/services/backup.py index 2541e52..4f9c88b 100644 --- a/src/fabledassistant/services/backup.py +++ b/src/fabledassistant/services/backup.py @@ -1,175 +1,369 @@ import logging +from datetime import date, datetime, timezone from sqlalchemy import select from sqlalchemy.orm import selectinload from fabledassistant.models import async_session from fabledassistant.models.conversation import Conversation, Message +from fabledassistant.models.milestone import Milestone from fabledassistant.models.note import Note +from fabledassistant.models.note_draft import NoteDraft +from fabledassistant.models.note_version import NoteVersion +from fabledassistant.models.project import Project +from fabledassistant.models.push_subscription import PushSubscription from fabledassistant.models.setting import Setting +from fabledassistant.models.task_log import TaskLog from fabledassistant.models.user import User logger = logging.getLogger(__name__) +def _dt(val: str | None) -> datetime: + return datetime.fromisoformat(val) if val else datetime.now(timezone.utc) + + +def _d(val: str | None) -> date | None: + return date.fromisoformat(val) if val else None + + +# --------------------------------------------------------------------------- +# Export +# --------------------------------------------------------------------------- + async def export_full_backup() -> dict: - """Export all users, notes, conversations+messages, and settings as JSON.""" + """Export all data as version-2 JSON backup.""" async with async_session() as session: users = (await session.execute(select(User))).scalars().all() + projects = (await session.execute(select(Project))).scalars().all() + milestones = (await session.execute(select(Milestone))).scalars().all() notes = (await session.execute(select(Note))).scalars().all() - conversations = ( - await session.execute( - select(Conversation).options(selectinload(Conversation.messages)) - ) - ).scalars().all() + task_logs = (await session.execute(select(TaskLog))).scalars().all() + note_drafts = (await session.execute(select(NoteDraft))).scalars().all() + note_versions = (await session.execute( + select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id) + )).scalars().all() + conversations = (await session.execute( + select(Conversation).options(selectinload(Conversation.messages)) + )).scalars().all() settings = (await session.execute(select(Setting))).scalars().all() + push_subs = (await session.execute(select(PushSubscription))).scalars().all() - return { - "version": 1, - "scope": "full", - "_security_notice": ( - "This backup contains hashed passwords. " - "Store it securely and restrict access." - ), - "users": [ - { - "id": u.id, - "username": u.username, - "email": u.email, - "password_hash": u.password_hash, - "role": u.role, - "created_at": u.created_at.isoformat(), - } - for u in users - ], - "notes": [ - { - "id": n.id, - "user_id": n.user_id, - "title": n.title, - "body": n.body, - "tags": n.tags or [], - "parent_id": n.parent_id, - "status": n.status, - "priority": n.priority, - "due_date": n.due_date.isoformat() if n.due_date else None, - "created_at": n.created_at.isoformat(), - "updated_at": n.updated_at.isoformat(), - } - for n in notes - ], - "conversations": [ - { - "id": c.id, - "user_id": c.user_id, - "title": c.title, - "model": c.model, - "created_at": c.created_at.isoformat(), - "updated_at": c.updated_at.isoformat(), - "messages": [ - { - "id": m.id, - "role": m.role, - "content": m.content, - "context_note_id": m.context_note_id, - "created_at": m.created_at.isoformat(), - } - for m in c.messages - ], - } - for c in conversations - ], - "settings": [ - { - "user_id": s.user_id, - "key": s.key, - "value": s.value, - } - for s in settings - ], - } + return { + "version": 2, + "scope": "full", + "exported_at": datetime.now(timezone.utc).isoformat(), + "_security_notice": ( + "This backup contains hashed passwords and push subscription keys. " + "Store it securely and restrict access." + ), + "users": [ + { + "id": u.id, + "username": u.username, + "email": u.email, + "password_hash": u.password_hash, + "oauth_sub": u.oauth_sub, + "role": u.role, + "session_version": u.session_version, + "created_at": u.created_at.isoformat(), + } + for u in users + ], + "projects": [ + { + "id": p.id, + "user_id": p.user_id, + "title": p.title, + "description": p.description, + "goal": p.goal, + "status": p.status, + "color": p.color, + "created_at": p.created_at.isoformat(), + "updated_at": p.updated_at.isoformat(), + } + for p in projects + ], + "milestones": [ + { + "id": m.id, + "user_id": m.user_id, + "project_id": m.project_id, + "title": m.title, + "description": m.description, + "status": m.status, + "order_index": m.order_index, + "created_at": m.created_at.isoformat(), + "updated_at": m.updated_at.isoformat(), + } + for m in milestones + ], + "notes": [ + { + "id": n.id, + "user_id": n.user_id, + "title": n.title, + "body": n.body, + "tags": n.tags or [], + "parent_id": n.parent_id, + "project_id": n.project_id, + "milestone_id": n.milestone_id, + "status": n.status, + "priority": n.priority, + "due_date": n.due_date.isoformat() if n.due_date else None, + "created_at": n.created_at.isoformat(), + "updated_at": n.updated_at.isoformat(), + } + for n in notes + ], + "task_logs": [ + { + "id": tl.id, + "user_id": tl.user_id, + "task_id": tl.task_id, + "content": tl.content, + "duration_minutes": tl.duration_minutes, + "created_at": tl.created_at.isoformat(), + "updated_at": tl.updated_at.isoformat(), + } + for tl in task_logs + ], + "note_drafts": [ + { + "id": nd.id, + "user_id": nd.user_id, + "note_id": nd.note_id, + "proposed_body": nd.proposed_body, + "original_body": nd.original_body, + "instruction": nd.instruction, + "scope": nd.scope, + "created_at": nd.created_at.isoformat(), + "updated_at": nd.updated_at.isoformat(), + } + for nd in note_drafts + ], + "note_versions": [ + { + "id": nv.id, + "user_id": nv.user_id, + "note_id": nv.note_id, + "title": nv.title, + "body": nv.body, + "tags": nv.tags or [], + "created_at": nv.created_at.isoformat(), + } + for nv in note_versions + ], + "conversations": [ + { + "id": c.id, + "user_id": c.user_id, + "title": c.title, + "model": c.model, + "created_at": c.created_at.isoformat(), + "updated_at": c.updated_at.isoformat(), + "messages": [ + { + "id": m.id, + "role": m.role, + "content": m.content, + "status": m.status, + "context_note_id": m.context_note_id, + "tool_calls": m.tool_calls, + "created_at": m.created_at.isoformat(), + } + for m in c.messages + ], + } + for c in conversations + ], + "settings": [ + {"user_id": s.user_id, "key": s.key, "value": s.value} + for s in settings + ], + "push_subscriptions": [ + { + "user_id": ps.user_id, + "endpoint": ps.endpoint, + "p256dh": ps.p256dh, + "auth": ps.auth, + "user_agent": ps.user_agent, + "created_at": ps.created_at.isoformat(), + } + for ps in push_subs + ], + } async def export_user_backup(user_id: int) -> dict: - """Export a single user's data as JSON.""" + """Export a single user's data as version-2 JSON backup.""" async with async_session() as session: user = await session.get(User, user_id) - notes = ( - await session.execute(select(Note).where(Note.user_id == user_id)) - ).scalars().all() - conversations = ( - await session.execute( - select(Conversation) - .options(selectinload(Conversation.messages)) - .where(Conversation.user_id == user_id) - ) - ).scalars().all() - settings = ( - await session.execute( - select(Setting).where(Setting.user_id == user_id) - ) - ).scalars().all() + projects = (await session.execute( + select(Project).where(Project.user_id == user_id) + )).scalars().all() + milestones = (await session.execute( + select(Milestone).where(Milestone.user_id == user_id) + )).scalars().all() + notes = (await session.execute( + select(Note).where(Note.user_id == user_id) + )).scalars().all() + task_logs = (await session.execute( + select(TaskLog).where(TaskLog.user_id == user_id) + )).scalars().all() + note_drafts = (await session.execute( + select(NoteDraft).where(NoteDraft.user_id == user_id) + )).scalars().all() + note_versions = (await session.execute( + select(NoteVersion).where(NoteVersion.user_id == user_id) + .order_by(NoteVersion.note_id, NoteVersion.id) + )).scalars().all() + conversations = (await session.execute( + select(Conversation) + .options(selectinload(Conversation.messages)) + .where(Conversation.user_id == user_id) + )).scalars().all() + settings = (await session.execute( + select(Setting).where(Setting.user_id == user_id) + )).scalars().all() - return { - "version": 1, - "scope": "user", - "user": { - "id": user.id, - "username": user.username, - "email": user.email, - "role": user.role, - "created_at": user.created_at.isoformat(), - } if user else None, - "notes": [ - { - "id": n.id, - "title": n.title, - "body": n.body, - "tags": n.tags or [], - "parent_id": n.parent_id, - "status": n.status, - "priority": n.priority, - "due_date": n.due_date.isoformat() if n.due_date else None, - "created_at": n.created_at.isoformat(), - "updated_at": n.updated_at.isoformat(), - } - for n in notes - ], - "conversations": [ - { - "id": c.id, - "title": c.title, - "model": c.model, - "created_at": c.created_at.isoformat(), - "updated_at": c.updated_at.isoformat(), - "messages": [ - { - "id": m.id, - "role": m.role, - "content": m.content, - "context_note_id": m.context_note_id, - "created_at": m.created_at.isoformat(), - } - for m in c.messages - ], - } - for c in conversations - ], - "settings": [ - {"key": s.key, "value": s.value} - for s in settings - ], - } + return { + "version": 2, + "scope": "user", + "exported_at": datetime.now(timezone.utc).isoformat(), + "user": { + "id": user.id, + "username": user.username, + "email": user.email, + "role": user.role, + "created_at": user.created_at.isoformat(), + } if user else None, + "projects": [ + { + "id": p.id, + "user_id": p.user_id, + "title": p.title, + "description": p.description, + "goal": p.goal, + "status": p.status, + "color": p.color, + "created_at": p.created_at.isoformat(), + "updated_at": p.updated_at.isoformat(), + } + for p in projects + ], + "milestones": [ + { + "id": m.id, + "user_id": m.user_id, + "project_id": m.project_id, + "title": m.title, + "description": m.description, + "status": m.status, + "order_index": m.order_index, + "created_at": m.created_at.isoformat(), + "updated_at": m.updated_at.isoformat(), + } + for m in milestones + ], + "notes": [ + { + "id": n.id, + "title": n.title, + "body": n.body, + "tags": n.tags or [], + "parent_id": n.parent_id, + "project_id": n.project_id, + "milestone_id": n.milestone_id, + "status": n.status, + "priority": n.priority, + "due_date": n.due_date.isoformat() if n.due_date else None, + "created_at": n.created_at.isoformat(), + "updated_at": n.updated_at.isoformat(), + } + for n in notes + ], + "task_logs": [ + { + "id": tl.id, + "task_id": tl.task_id, + "content": tl.content, + "duration_minutes": tl.duration_minutes, + "created_at": tl.created_at.isoformat(), + "updated_at": tl.updated_at.isoformat(), + } + for tl in task_logs + ], + "note_drafts": [ + { + "id": nd.id, + "note_id": nd.note_id, + "proposed_body": nd.proposed_body, + "original_body": nd.original_body, + "instruction": nd.instruction, + "scope": nd.scope, + "created_at": nd.created_at.isoformat(), + "updated_at": nd.updated_at.isoformat(), + } + for nd in note_drafts + ], + "note_versions": [ + { + "id": nv.id, + "note_id": nv.note_id, + "title": nv.title, + "body": nv.body, + "tags": nv.tags or [], + "created_at": nv.created_at.isoformat(), + } + for nv in note_versions + ], + "conversations": [ + { + "id": c.id, + "title": c.title, + "model": c.model, + "created_at": c.created_at.isoformat(), + "updated_at": c.updated_at.isoformat(), + "messages": [ + { + "id": m.id, + "role": m.role, + "content": m.content, + "status": m.status, + "context_note_id": m.context_note_id, + "tool_calls": m.tool_calls, + "created_at": m.created_at.isoformat(), + } + for m in c.messages + ], + } + for c in conversations + ], + "settings": [ + {"key": s.key, "value": s.value} + for s in settings + ], + } +# --------------------------------------------------------------------------- +# Restore +# --------------------------------------------------------------------------- + async def restore_full_backup(data: dict) -> dict: - """Restore from a full backup JSON. Returns stats about what was restored.""" - from datetime import date, datetime, timezone + """Restore from backup JSON. Dispatches by version.""" + version = data.get("version", 1) + if version == 1: + return await _restore_v1(data) + return await _restore_v2(data) + +async def _restore_v1(data: dict) -> dict: + """Restore legacy v1 backup (original format).""" stats = {"users": 0, "notes": 0, "conversations": 0, "messages": 0, "settings": 0} async with async_session() as session: - # Restore users user_id_map: dict[int, int] = {} for u_data in data.get("users", []): old_id = u_data["id"] @@ -178,34 +372,30 @@ async def restore_full_backup(data: dict) -> dict: email=u_data.get("email"), password_hash=u_data["password_hash"], role=u_data.get("role", "user"), - created_at=datetime.fromisoformat(u_data["created_at"]) if u_data.get("created_at") else datetime.now(timezone.utc), + created_at=_dt(u_data.get("created_at")), ) session.add(user) await session.flush() user_id_map[old_id] = user.id stats["users"] += 1 - # Restore notes note_id_map: dict[int, int] = {} for n_data in data.get("notes", []): old_id = n_data.get("id") mapped_user_id = user_id_map.get(n_data.get("user_id", 0)) if mapped_user_id is None: continue - due = None - if n_data.get("due_date"): - due = date.fromisoformat(n_data["due_date"]) note = Note( user_id=mapped_user_id, title=n_data.get("title", ""), body=n_data.get("body", ""), tags=n_data.get("tags", []), - parent_id=n_data.get("parent_id"), + parent_id=None, # patched below status=n_data.get("status"), priority=n_data.get("priority"), - due_date=due, - created_at=datetime.fromisoformat(n_data["created_at"]) if n_data.get("created_at") else datetime.now(timezone.utc), - updated_at=datetime.fromisoformat(n_data["updated_at"]) if n_data.get("updated_at") else datetime.now(timezone.utc), + due_date=_d(n_data.get("due_date")), + created_at=_dt(n_data.get("created_at")), + updated_at=_dt(n_data.get("updated_at")), ) session.add(note) await session.flush() @@ -213,7 +403,15 @@ async def restore_full_backup(data: dict) -> dict: note_id_map[old_id] = note.id stats["notes"] += 1 - # Restore conversations + messages + # Patch parent_id now that all notes have new IDs + for n_data in data.get("notes", []): + old_id = n_data.get("id") + old_parent = n_data.get("parent_id") + if old_id and old_parent and old_id in note_id_map and old_parent in note_id_map: + note_row = await session.get(Note, note_id_map[old_id]) + if note_row: + note_row.parent_id = note_id_map[old_parent] + for c_data in data.get("conversations", []): mapped_user_id = user_id_map.get(c_data.get("user_id", 0)) if mapped_user_id is None: @@ -222,38 +420,233 @@ async def restore_full_backup(data: dict) -> dict: user_id=mapped_user_id, title=c_data.get("title", ""), model=c_data.get("model", ""), - created_at=datetime.fromisoformat(c_data["created_at"]) if c_data.get("created_at") else datetime.now(timezone.utc), - updated_at=datetime.fromisoformat(c_data["updated_at"]) if c_data.get("updated_at") else datetime.now(timezone.utc), + created_at=_dt(c_data.get("created_at")), + updated_at=_dt(c_data.get("updated_at")), ) session.add(conv) await session.flush() stats["conversations"] += 1 - for m_data in c_data.get("messages", []): msg = Message( conversation_id=conv.id, role=m_data["role"], content=m_data.get("content", ""), - context_note_id=note_id_map.get(m_data.get("context_note_id")) if m_data.get("context_note_id") else None, - created_at=datetime.fromisoformat(m_data["created_at"]) if m_data.get("created_at") else datetime.now(timezone.utc), + context_note_id=note_id_map.get(m_data["context_note_id"]) if m_data.get("context_note_id") else None, + created_at=_dt(m_data.get("created_at")), ) session.add(msg) stats["messages"] += 1 - # Restore settings for s_data in data.get("settings", []): mapped_user_id = user_id_map.get(s_data.get("user_id", 0)) if mapped_user_id is None: continue - setting = Setting( - user_id=mapped_user_id, - key=s_data["key"], - value=s_data.get("value", ""), - ) - session.add(setting) + session.add(Setting(user_id=mapped_user_id, key=s_data["key"], value=s_data.get("value", ""))) stats["settings"] += 1 await session.commit() - logger.info("Restored backup: %s", stats) + logger.info("Restored v1 backup: %s", stats) + return stats + + +async def _restore_v2(data: dict) -> dict: + """Restore v2 backup with full FK re-mapping.""" + stats: dict[str, int] = { + "users": 0, "projects": 0, "milestones": 0, "notes": 0, + "task_logs": 0, "note_drafts": 0, "note_versions": 0, + "conversations": 0, "messages": 0, "settings": 0, + } + + async with async_session() as session: + user_id_map: dict[int, int] = {} + project_id_map: dict[int, int] = {} + milestone_id_map: dict[int, int] = {} + note_id_map: dict[int, int] = {} + + # 1. Users + for u_data in data.get("users", []): + old_id = u_data["id"] + user = User( + username=u_data["username"], + email=u_data.get("email"), + password_hash=u_data.get("password_hash"), + oauth_sub=u_data.get("oauth_sub"), + role=u_data.get("role", "user"), + session_version=u_data.get("session_version", 1), + created_at=_dt(u_data.get("created_at")), + ) + session.add(user) + await session.flush() + user_id_map[old_id] = user.id + stats["users"] += 1 + + # 2. Projects + for p_data in data.get("projects", []): + mapped_uid = user_id_map.get(p_data.get("user_id", 0)) + if mapped_uid is None: + continue + proj = Project( + user_id=mapped_uid, + title=p_data.get("title", ""), + description=p_data.get("description", ""), + goal=p_data.get("goal", ""), + status=p_data.get("status", "active"), + color=p_data.get("color"), + created_at=_dt(p_data.get("created_at")), + updated_at=_dt(p_data.get("updated_at")), + ) + session.add(proj) + await session.flush() + project_id_map[p_data["id"]] = proj.id + stats["projects"] += 1 + + # 3. Milestones + for m_data in data.get("milestones", []): + mapped_uid = user_id_map.get(m_data.get("user_id", 0)) + mapped_pid = project_id_map.get(m_data.get("project_id", 0)) + if mapped_uid is None or mapped_pid is None: + continue + ms = Milestone( + user_id=mapped_uid, + project_id=mapped_pid, + title=m_data.get("title", ""), + description=m_data.get("description"), + status=m_data.get("status", "active"), + order_index=m_data.get("order_index", 0), + created_at=_dt(m_data.get("created_at")), + updated_at=_dt(m_data.get("updated_at")), + ) + session.add(ms) + await session.flush() + milestone_id_map[m_data["id"]] = ms.id + stats["milestones"] += 1 + + # 4a. Notes — first pass (no parent_id yet) + notes_with_parents: list[tuple[int, int]] = [] # (new_note_id, old_parent_id) + for n_data in data.get("notes", []): + mapped_uid = user_id_map.get(n_data.get("user_id", 0)) + if mapped_uid is None: + continue + note = Note( + user_id=mapped_uid, + title=n_data.get("title", ""), + body=n_data.get("body", ""), + tags=n_data.get("tags", []), + parent_id=None, + project_id=project_id_map.get(n_data["project_id"]) if n_data.get("project_id") else None, + milestone_id=milestone_id_map.get(n_data["milestone_id"]) if n_data.get("milestone_id") else None, + status=n_data.get("status"), + priority=n_data.get("priority"), + due_date=_d(n_data.get("due_date")), + created_at=_dt(n_data.get("created_at")), + updated_at=_dt(n_data.get("updated_at")), + ) + session.add(note) + await session.flush() + note_id_map[n_data["id"]] = note.id + if n_data.get("parent_id"): + notes_with_parents.append((note.id, n_data["parent_id"])) + stats["notes"] += 1 + + # 4b. Patch parent_id + for new_note_id, old_parent_id in notes_with_parents: + new_parent_id = note_id_map.get(old_parent_id) + if new_parent_id: + note_row = await session.get(Note, new_note_id) + if note_row: + note_row.parent_id = new_parent_id + + # 5. TaskLogs + for tl_data in data.get("task_logs", []): + mapped_uid = user_id_map.get(tl_data.get("user_id", 0)) + mapped_tid = note_id_map.get(tl_data.get("task_id", 0)) + if mapped_uid is None or mapped_tid is None: + continue + tl = TaskLog( + user_id=mapped_uid, + task_id=mapped_tid, + content=tl_data.get("content", ""), + duration_minutes=tl_data.get("duration_minutes"), + created_at=_dt(tl_data.get("created_at")), + updated_at=_dt(tl_data.get("updated_at")), + ) + session.add(tl) + stats["task_logs"] += 1 + + # 6. NoteDrafts + for nd_data in data.get("note_drafts", []): + mapped_uid = user_id_map.get(nd_data.get("user_id", 0)) + mapped_nid = note_id_map.get(nd_data.get("note_id", 0)) + if mapped_uid is None or mapped_nid is None: + continue + nd = NoteDraft( + user_id=mapped_uid, + note_id=mapped_nid, + proposed_body=nd_data.get("proposed_body", ""), + original_body=nd_data.get("original_body", ""), + instruction=nd_data.get("instruction", ""), + scope=nd_data.get("scope", "document"), + created_at=_dt(nd_data.get("created_at")), + updated_at=_dt(nd_data.get("updated_at")), + ) + session.add(nd) + stats["note_drafts"] += 1 + + # 7. NoteVersions + for nv_data in data.get("note_versions", []): + mapped_uid = user_id_map.get(nv_data.get("user_id", 0)) + mapped_nid = note_id_map.get(nv_data.get("note_id", 0)) + if mapped_uid is None or mapped_nid is None: + continue + nv = NoteVersion( + user_id=mapped_uid, + note_id=mapped_nid, + title=nv_data.get("title", ""), + body=nv_data.get("body", ""), + tags=nv_data.get("tags", []), + created_at=_dt(nv_data.get("created_at")), + ) + session.add(nv) + stats["note_versions"] += 1 + + # 8. Conversations + Messages + for c_data in data.get("conversations", []): + mapped_uid = user_id_map.get(c_data.get("user_id", 0)) + if mapped_uid is None: + continue + conv = Conversation( + user_id=mapped_uid, + title=c_data.get("title", ""), + model=c_data.get("model", ""), + created_at=_dt(c_data.get("created_at")), + updated_at=_dt(c_data.get("updated_at")), + ) + session.add(conv) + await session.flush() + stats["conversations"] += 1 + for m_data in c_data.get("messages", []): + msg = Message( + conversation_id=conv.id, + role=m_data["role"], + content=m_data.get("content", ""), + status=m_data.get("status", "complete"), + context_note_id=note_id_map.get(m_data["context_note_id"]) if m_data.get("context_note_id") else None, + tool_calls=m_data.get("tool_calls"), + created_at=_dt(m_data.get("created_at")), + ) + session.add(msg) + stats["messages"] += 1 + + # 9. Settings + for s_data in data.get("settings", []): + mapped_uid = user_id_map.get(s_data.get("user_id", 0)) + if mapped_uid is None: + continue + session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", ""))) + stats["settings"] += 1 + + await session.commit() + + logger.info("Restored v2 backup: %s", stats) return stats