FabledGrimoire/src/scribe/services/backup.py

import logging
from datetime import date, datetime, timezone

from sqlalchemy import or_, select

from scribe.models import async_session
from scribe.models.milestone import Milestone
from scribe.models.note import Note
from scribe.models.note_draft import NoteDraft
from scribe.models.note_version import NoteVersion
from scribe.models.design_system import DesignSystem, DesignToken
from scribe.models.note_usage import NoteUsageEvent
from scribe.models.project import Project
from scribe.models.repo_binding import RepoBinding
from scribe.models.rulebook import (
    Rule,
    Rulebook,
    RulebookTopic,
    project_rule_suppressions,
    project_rulebook_subscriptions,
    project_topic_suppressions,
)
from scribe.models.setting import Setting
from scribe.models.system import RecordSystem, System
from scribe.models.task_log import TaskLog
from scribe.models.user import User

logger = logging.getLogger(__name__)

# Backup format version. v3 (2026-06) added rulebooks/topics/rules + their
# project subscription/suppression join tables. v4 (2026-07) dropped events
# when the calendar surface was retired — old v3 events are skipped on restore.
# v5 (2026-08) added the six tables that had accumulated outside the backup
# entirely (#2293), and the coverage guard that stops the seventh.
# Bump when the serialized schema changes.
BACKUP_VERSION = 5

# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
# below, these two lists must together account for the entire schema — which is
# what tests/test_services_backup.py asserts against Base.metadata.
#
# The point is the ABSENCE case. A new table gets a model and a migration, both
# of which fail loudly if wrong, and then silently never gets a backup section:
# no error, no warning, and a restore that reports success. Naming the coverage
# explicitly turns "someone forgot" into a failing test (#2293).
_BACKED_UP = [
    "users", "projects", "milestones", "notes", "task_logs", "note_drafts",
    "note_versions", "settings", "rulebooks", "rulebook_topics", "rules",
    "project_rulebook_subscriptions", "project_rule_suppressions",
    "project_topic_suppressions",
    # v5 (2026-08): the five-year gap this list was written to stop.
    "systems", "record_systems", "design_systems", "design_tokens",
    "note_usage_events", "repo_bindings",
]

# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
# explicit rather than silent. ACL (groups/shares) is a coherent follow-up;
# note_embeddings are derived (regenerated from note bodies); api_keys are
# sensitive credentials; retrieval_logs is observational telemetry that nothing
# reads for correctness and that grows per query; the rest are
# transient/operational.
#
# REAL table names, deliberately. This list used to read "embeddings",
# "invitations", "password_resets" — none of which are tables — so it looked
# like coverage while naming nothing the schema could confirm.
_NOT_INCLUDED = [
    "groups", "group_memberships", "project_shares", "note_shares",
    "api_keys", "note_embeddings", "app_logs", "notifications",
    "invitation_tokens", "password_reset_tokens", "user_profiles",
    "retrieval_logs",
]


def _dt(val: str | None) -> datetime:
    return datetime.fromisoformat(val) if val else datetime.now(timezone.utc)


def _d(val: str | None) -> date | None:
    return date.fromisoformat(val) if val else None


def _subscription_rows(rows) -> list[dict]:
    return [{"project_id": r.project_id, "rulebook_id": r.rulebook_id} for r in rows]


def _rule_suppression_rows(rows) -> list[dict]:
    return [{"project_id": r.project_id, "rule_id": r.rule_id} for r in rows]


def _topic_suppression_rows(rows) -> list[dict]:
    return [{"project_id": r.project_id, "topic_id": r.topic_id} for r in rows]


# The v5 sections. Pure row-builders like the join-table helpers above, for the
# same reason: CI has no database, so a serialiser that is a plain function is
# one that can actually be tested.

def _system_rows(rows) -> list[dict]:
    return [
        {
            "id": r.id, "user_id": r.user_id, "project_id": r.project_id,
            "name": r.name, "description": r.description, "color": r.color,
            "status": r.status, "order_index": r.order_index,
        }
        for r in rows
    ]


def _record_system_rows(rows) -> list[dict]:
    return [{"note_id": r.note_id, "system_id": r.system_id} for r in rows]


def _design_system_rows(rows) -> list[dict]:
    return [
        {
            "id": r.id, "owner_user_id": r.owner_user_id, "title": r.title,
            "description": r.description, "guidance": r.guidance,
            "parent_id": r.parent_id,
        }
        for r in rows
    ]


def _design_token_rows(rows) -> list[dict]:
    return [
        {
            "id": r.id, "design_system_id": r.design_system_id, "name": r.name,
            "value_by_mode": r.value_by_mode or {},
            "group_name": r.group_name, "purpose": r.purpose,
            "rationale": r.rationale, "supersedes": r.supersedes or [],
            "order_index": r.order_index,
        }
        for r in rows
    ]


def _usage_event_rows(rows) -> list[dict]:
    return [
        {
            "user_id": r.user_id, "note_id": r.note_id, "event": r.event,
            "source": r.source,
            "created_at": r.created_at.isoformat() if r.created_at else None,
        }
        for r in rows
    ]


def _repo_binding_rows(rows) -> list[dict]:
    return [
        {"user_id": r.user_id, "project_id": r.project_id, "repo_key": r.repo_key}
        for r in rows
    ]


# ---------------------------------------------------------------------------
# Export
# ---------------------------------------------------------------------------

async def export_full_backup() -> dict:
    """Export all data as a version-5 JSON backup."""
    async with async_session() as session:
        users = (await session.execute(select(User))).scalars().all()
        projects = (await session.execute(select(Project))).scalars().all()
        milestones = (await session.execute(select(Milestone))).scalars().all()
        notes = (await session.execute(select(Note))).scalars().all()
        task_logs = (await session.execute(select(TaskLog))).scalars().all()
        note_drafts = (await session.execute(select(NoteDraft))).scalars().all()
        note_versions = (await session.execute(
            select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id)
        )).scalars().all()
        settings = (await session.execute(select(Setting))).scalars().all()
        systems = (await session.execute(select(System))).scalars().all()
        record_systems = (await session.execute(select(RecordSystem))).scalars().all()
        # Parent-first, so a restore can resolve parent_id as it goes rather
        # than needing a second pass — the self-FK is the only ordering
        # constraint in this payload.
        design_systems = (await session.execute(
            select(DesignSystem).order_by(DesignSystem.parent_id.nullsfirst(),
                                          DesignSystem.id)
        )).scalars().all()
        design_tokens = (await session.execute(select(DesignToken))).scalars().all()
        usage_events = (await session.execute(select(NoteUsageEvent))).scalars().all()
        repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
        rulebooks = (await session.execute(select(Rulebook))).scalars().all()
        topics = (await session.execute(select(RulebookTopic))).scalars().all()
        rules = (await session.execute(select(Rule))).scalars().all()
        subscriptions = (await session.execute(
            select(project_rulebook_subscriptions)
        )).all()
        rule_suppressions = (await session.execute(
            select(project_rule_suppressions)
        )).all()
        topic_suppressions = (await session.execute(
            select(project_topic_suppressions)
        )).all()

    return {
        "version": BACKUP_VERSION,
        "scope": "full",
        "exported_at": datetime.now(timezone.utc).isoformat(),
        "_security_notice": (
            "This backup contains hashed passwords. "
            "Store it securely and restrict access."
        ),
        "_not_included": _NOT_INCLUDED,
        "users": [
            {
                "id": u.id,
                "username": u.username,
                "email": u.email,
                "password_hash": u.password_hash,
                "oauth_sub": u.oauth_sub,
                "role": u.role,
                "session_version": u.session_version,
                "created_at": u.created_at.isoformat(),
            }
            for u in users
        ],
        "projects": [
            {
                "id": p.id,
                "user_id": p.user_id,
                "title": p.title,
                "description": p.description,
                "goal": p.goal,
                "status": p.status,
                "color": p.color,
                "created_at": p.created_at.isoformat(),
                "updated_at": p.updated_at.isoformat(),
            }
            for p in projects
        ],
        "milestones": [
            {
                "id": m.id,
                "user_id": m.user_id,
                "project_id": m.project_id,
                "title": m.title,
                "description": m.description,
                "status": m.status,
                "order_index": m.order_index,
                "created_at": m.created_at.isoformat(),
                "updated_at": m.updated_at.isoformat(),
            }
            for m in milestones
        ],
        "notes": [
            {
                "id": n.id,
                "user_id": n.user_id,
                "title": n.title,
                "body": n.body,
                "tags": n.tags or [],
                "parent_id": n.parent_id,
                "project_id": n.project_id,
                "milestone_id": n.milestone_id,
                "status": n.status,
                "priority": n.priority,
                "due_date": n.due_date.isoformat() if n.due_date else None,
                "created_at": n.created_at.isoformat(),
                "updated_at": n.updated_at.isoformat(),
            }
            for n in notes
        ],
        "task_logs": [
            {
                "id": tl.id,
                "user_id": tl.user_id,
                "task_id": tl.task_id,
                "content": tl.content,
                "duration_minutes": tl.duration_minutes,
                "created_at": tl.created_at.isoformat(),
                "updated_at": tl.updated_at.isoformat(),
            }
            for tl in task_logs
        ],
        "note_drafts": [
            {
                "id": nd.id,
                "user_id": nd.user_id,
                "note_id": nd.note_id,
                "proposed_body": nd.proposed_body,
                "original_body": nd.original_body,
                "instruction": nd.instruction,
                "scope": nd.scope,
                "created_at": nd.created_at.isoformat(),
                "updated_at": nd.updated_at.isoformat(),
            }
            for nd in note_drafts
        ],
        "note_versions": [
            {
                "id": nv.id,
                "user_id": nv.user_id,
                "note_id": nv.note_id,
                "title": nv.title,
                "body": nv.body,
                "tags": nv.tags or [],
                "pin_kind": nv.pin_kind,
                "pin_label": nv.pin_label,
                "created_at": nv.created_at.isoformat(),
            }
            for nv in note_versions
        ],
        "settings": [
            {"user_id": s.user_id, "key": s.key, "value": s.value}
            for s in settings
        ],
        "rulebooks": [
            {
                "id": rb.id,
                "owner_user_id": rb.owner_user_id,
                "title": rb.title,
                "description": rb.description,
                "always_on": rb.always_on,
                "created_at": rb.created_at.isoformat(),
                "updated_at": rb.updated_at.isoformat(),
            }
            for rb in rulebooks
        ],
        "rulebook_topics": [
            {
                "id": t.id,
                "rulebook_id": t.rulebook_id,
                "title": t.title,
                "description": t.description,
                "order_index": t.order_index,
                "created_at": t.created_at.isoformat(),
                "updated_at": t.updated_at.isoformat(),
            }
            for t in topics
        ],
        "rules": [
            {
                "id": r.id,
                "topic_id": r.topic_id,
                "project_id": r.project_id,
                "title": r.title,
                "statement": r.statement,
                "why": r.why,
                "how_to_apply": r.how_to_apply,
                "order_index": r.order_index,
                "created_at": r.created_at.isoformat(),
                "updated_at": r.updated_at.isoformat(),
            }
            for r in rules
        ],
        "rulebook_subscriptions": _subscription_rows(subscriptions),
        "rule_suppressions": _rule_suppression_rows(rule_suppressions),
        "topic_suppressions": _topic_suppression_rows(topic_suppressions),
        "systems": _system_rows(systems),
        "record_systems": _record_system_rows(record_systems),
        "design_systems": _design_system_rows(design_systems),
        "design_tokens": _design_token_rows(design_tokens),
        "note_usage_events": _usage_event_rows(usage_events),
        "repo_bindings": _repo_binding_rows(repo_bindings),
    }


async def export_user_backup(user_id: int) -> dict:
    """Export a single user's data as a version-5 JSON backup."""
    async with async_session() as session:
        user = await session.get(User, user_id)
        projects = (await session.execute(
            select(Project).where(Project.user_id == user_id)
        )).scalars().all()
        project_ids = [p.id for p in projects]
        milestones = (await session.execute(
            select(Milestone).where(Milestone.user_id == user_id)
        )).scalars().all()
        notes = (await session.execute(
            select(Note).where(Note.user_id == user_id)
        )).scalars().all()
        task_logs = (await session.execute(
            select(TaskLog).where(TaskLog.user_id == user_id)
        )).scalars().all()
        note_drafts = (await session.execute(
            select(NoteDraft).where(NoteDraft.user_id == user_id)
        )).scalars().all()
        note_versions = (await session.execute(
            select(NoteVersion).where(NoteVersion.user_id == user_id)
            .order_by(NoteVersion.note_id, NoteVersion.id)
        )).scalars().all()
        settings = (await session.execute(
            select(Setting).where(Setting.user_id == user_id)
        )).scalars().all()
        systems = (await session.execute(
            select(System).where(System.user_id == user_id)
        )).scalars().all()
        system_ids = [sy.id for sy in systems]
        note_ids = [n.id for n in notes]
        # Scoped by the user's SYSTEMS, not their notes: a shared note carrying
        # this user's system tag belongs in their backup, and a note of theirs
        # tagged with someone else's system does not — that row is the other
        # user's to keep.
        record_systems = (await session.execute(
            select(RecordSystem).where(RecordSystem.system_id.in_(system_ids))
        )).scalars().all() if system_ids else []
        design_systems = (await session.execute(
            select(DesignSystem).where(DesignSystem.owner_user_id == user_id)
            .order_by(DesignSystem.parent_id.nullsfirst(), DesignSystem.id)
        )).scalars().all()
        ds_ids = [d.id for d in design_systems]
        design_tokens = (await session.execute(
            select(DesignToken).where(DesignToken.design_system_id.in_(ds_ids))
        )).scalars().all() if ds_ids else []
        usage_events = (await session.execute(
            select(NoteUsageEvent).where(NoteUsageEvent.note_id.in_(note_ids))
        )).scalars().all() if note_ids else []
        repo_bindings = (await session.execute(
            select(RepoBinding).where(RepoBinding.user_id == user_id)
        )).scalars().all()
        rulebooks = (await session.execute(
            select(Rulebook).where(Rulebook.owner_user_id == user_id)
        )).scalars().all()
        rulebook_ids = [rb.id for rb in rulebooks]
        topics = (await session.execute(
            select(RulebookTopic).where(RulebookTopic.rulebook_id.in_(rulebook_ids))
        )).scalars().all() if rulebook_ids else []
        topic_ids = [t.id for t in topics]
        # Rules owned by the user = rules in the user's topics OR project-rules
        # on the user's projects.
        rule_filters = []
        if topic_ids:
            rule_filters.append(Rule.topic_id.in_(topic_ids))
        if project_ids:
            rule_filters.append(Rule.project_id.in_(project_ids))
        rules = (await session.execute(
            select(Rule).where(or_(*rule_filters))
        )).scalars().all() if rule_filters else []
        if project_ids:
            subscriptions = (await session.execute(
                select(project_rulebook_subscriptions).where(
                    project_rulebook_subscriptions.c.project_id.in_(project_ids)
                )
            )).all()
            rule_suppressions = (await session.execute(
                select(project_rule_suppressions).where(
                    project_rule_suppressions.c.project_id.in_(project_ids)
                )
            )).all()
            topic_suppressions = (await session.execute(
                select(project_topic_suppressions).where(
                    project_topic_suppressions.c.project_id.in_(project_ids)
                )
            )).all()
        else:
            subscriptions = rule_suppressions = topic_suppressions = []

    return {
        "version": BACKUP_VERSION,
        "scope": "user",
        "exported_at": datetime.now(timezone.utc).isoformat(),
        "_not_included": _NOT_INCLUDED,
        "user": {
            "id": user.id,
            "username": user.username,
            "email": user.email,
            "role": user.role,
            "created_at": user.created_at.isoformat(),
        } if user else None,
        "projects": [
            {
                "id": p.id,
                "user_id": p.user_id,
                "title": p.title,
                "description": p.description,
                "goal": p.goal,
                "status": p.status,
                "color": p.color,
                "created_at": p.created_at.isoformat(),
                "updated_at": p.updated_at.isoformat(),
            }
            for p in projects
        ],
        "milestones": [
            {
                "id": m.id,
                "user_id": m.user_id,
                "project_id": m.project_id,
                "title": m.title,
                "description": m.description,
                "status": m.status,
                "order_index": m.order_index,
                "created_at": m.created_at.isoformat(),
                "updated_at": m.updated_at.isoformat(),
            }
            for m in milestones
        ],
        "notes": [
            {
                "id": n.id,
                "user_id": n.user_id,
                "title": n.title,
                "body": n.body,
                "tags": n.tags or [],
                "parent_id": n.parent_id,
                "project_id": n.project_id,
                "milestone_id": n.milestone_id,
                "status": n.status,
                "priority": n.priority,
                "due_date": n.due_date.isoformat() if n.due_date else None,
                "created_at": n.created_at.isoformat(),
                "updated_at": n.updated_at.isoformat(),
            }
            for n in notes
        ],
        "task_logs": [
            {
                "id": tl.id,
                "user_id": tl.user_id,
                "task_id": tl.task_id,
                "content": tl.content,
                "duration_minutes": tl.duration_minutes,
                "created_at": tl.created_at.isoformat(),
                "updated_at": tl.updated_at.isoformat(),
            }
            for tl in task_logs
        ],
        "note_drafts": [
            {
                "id": nd.id,
                "user_id": nd.user_id,
                "note_id": nd.note_id,
                "proposed_body": nd.proposed_body,
                "original_body": nd.original_body,
                "instruction": nd.instruction,
                "scope": nd.scope,
                "created_at": nd.created_at.isoformat(),
                "updated_at": nd.updated_at.isoformat(),
            }
            for nd in note_drafts
        ],
        "note_versions": [
            {
                "id": nv.id,
                "user_id": nv.user_id,
                "note_id": nv.note_id,
                "title": nv.title,
                "body": nv.body,
                "tags": nv.tags or [],
                "pin_kind": nv.pin_kind,
                "pin_label": nv.pin_label,
                "created_at": nv.created_at.isoformat(),
            }
            for nv in note_versions
        ],
        "settings": [
            {"user_id": s.user_id, "key": s.key, "value": s.value}
            for s in settings
        ],
        "rulebooks": [
            {
                "id": rb.id,
                "owner_user_id": rb.owner_user_id,
                "title": rb.title,
                "description": rb.description,
                "always_on": rb.always_on,
                "created_at": rb.created_at.isoformat(),
                "updated_at": rb.updated_at.isoformat(),
            }
            for rb in rulebooks
        ],
        "rulebook_topics": [
            {
                "id": t.id,
                "rulebook_id": t.rulebook_id,
                "title": t.title,
                "description": t.description,
                "order_index": t.order_index,
                "created_at": t.created_at.isoformat(),
                "updated_at": t.updated_at.isoformat(),
            }
            for t in topics
        ],
        "rules": [
            {
                "id": r.id,
                "topic_id": r.topic_id,
                "project_id": r.project_id,
                "title": r.title,
                "statement": r.statement,
                "why": r.why,
                "how_to_apply": r.how_to_apply,
                "order_index": r.order_index,
                "created_at": r.created_at.isoformat(),
                "updated_at": r.updated_at.isoformat(),
            }
            for r in rules
        ],
        "rulebook_subscriptions": _subscription_rows(subscriptions),
        "rule_suppressions": _rule_suppression_rows(rule_suppressions),
        "topic_suppressions": _topic_suppression_rows(topic_suppressions),
        "systems": _system_rows(systems),
        "record_systems": _record_system_rows(record_systems),
        "design_systems": _design_system_rows(design_systems),
        "design_tokens": _design_token_rows(design_tokens),
        "note_usage_events": _usage_event_rows(usage_events),
        "repo_bindings": _repo_binding_rows(repo_bindings),
    }


# ---------------------------------------------------------------------------
# Restore
# ---------------------------------------------------------------------------

async def restore_full_backup(data: dict) -> dict:
    """Restore from backup JSON. Dispatches by version."""
    version = data.get("version", 1)
    if version == 1:
        return await _restore_v1(data)
    # v2 and v3 share one path; v3-only sections are guarded by data.get so a
    # v2 payload (without them) restores cleanly.
    return await _restore_v2(data)


async def _restore_v1(data: dict) -> dict:
    """Restore legacy v1 backup (original format).

    Pre-pivot v1 backups included conversations + messages; those are
    skipped during restore now that the chat subsystem is gone.
    """
    stats = {"users": 0, "notes": 0, "settings": 0}

    async with async_session() as session:
        user_id_map: dict[int, int] = {}
        for u_data in data.get("users", []):
            old_id = u_data["id"]
            user = User(
                username=u_data["username"],
                email=u_data.get("email"),
                password_hash=u_data["password_hash"],
                role=u_data.get("role", "user"),
                created_at=_dt(u_data.get("created_at")),
            )
            session.add(user)
            await session.flush()
            user_id_map[old_id] = user.id
            stats["users"] += 1

        note_id_map: dict[int, int] = {}
        for n_data in data.get("notes", []):
            old_id = n_data.get("id")
            mapped_user_id = user_id_map.get(n_data.get("user_id", 0))
            if mapped_user_id is None:
                continue
            note = Note(
                user_id=mapped_user_id,
                title=n_data.get("title", ""),
                body=n_data.get("body", ""),
                tags=n_data.get("tags", []),
                parent_id=None,  # patched below
                status=n_data.get("status"),
                priority=n_data.get("priority"),
                due_date=_d(n_data.get("due_date")),
                created_at=_dt(n_data.get("created_at")),
                updated_at=_dt(n_data.get("updated_at")),
            )
            session.add(note)
            await session.flush()
            if old_id is not None:
                note_id_map[old_id] = note.id
            stats["notes"] += 1

        # Patch parent_id now that all notes have new IDs
        for n_data in data.get("notes", []):
            old_id = n_data.get("id")
            old_parent = n_data.get("parent_id")
            if old_id and old_parent and old_id in note_id_map and old_parent in note_id_map:
                note_row = await session.get(Note, note_id_map[old_id])
                if note_row:
                    note_row.parent_id = note_id_map[old_parent]

        for s_data in data.get("settings", []):
            mapped_user_id = user_id_map.get(s_data.get("user_id", 0))
            if mapped_user_id is None:
                continue
            session.add(Setting(user_id=mapped_user_id, key=s_data["key"], value=s_data.get("value", "")))
            stats["settings"] += 1

        await session.commit()

    logger.info("Restored v1 backup: %s", stats)
    return stats


async def _restore_v2(data: dict) -> dict:
    """Restore v2/v3 backup with full FK re-mapping.

    Conversations, push subscriptions, and (as of v4) events in older backups
    are silently skipped — those subsystems were removed. v3+ sections
    (rulebooks/topics/rules/join-tables) are guarded by data.get so a v2
    payload restores without them.
    """
    stats: dict[str, int] = {
        "users": 0, "projects": 0, "milestones": 0, "notes": 0,
        "task_logs": 0, "note_drafts": 0, "note_versions": 0,
        "settings": 0, "rulebooks": 0, "rulebook_topics": 0, "rules": 0,
        "rulebook_subscriptions": 0, "rule_suppressions": 0,
        "topic_suppressions": 0,
        "systems": 0, "record_systems": 0, "design_systems": 0,
        "design_tokens": 0, "note_usage_events": 0, "repo_bindings": 0,
    }

    async with async_session() as session:
        user_id_map: dict[int, int] = {}
        project_id_map: dict[int, int] = {}
        milestone_id_map: dict[int, int] = {}
        note_id_map: dict[int, int] = {}
        rulebook_id_map: dict[int, int] = {}
        topic_id_map: dict[int, int] = {}
        rule_id_map: dict[int, int] = {}

        # 1. Users
        for u_data in data.get("users", []):
            old_id = u_data["id"]
            user = User(
                username=u_data["username"],
                email=u_data.get("email"),
                password_hash=u_data.get("password_hash"),
                oauth_sub=u_data.get("oauth_sub"),
                role=u_data.get("role", "user"),
                session_version=u_data.get("session_version", 1),
                created_at=_dt(u_data.get("created_at")),
            )
            session.add(user)
            await session.flush()
            user_id_map[old_id] = user.id
            stats["users"] += 1

        # 2. Projects
        for p_data in data.get("projects", []):
            mapped_uid = user_id_map.get(p_data.get("user_id", 0))
            if mapped_uid is None:
                continue
            proj = Project(
                user_id=mapped_uid,
                title=p_data.get("title", ""),
                description=p_data.get("description", ""),
                goal=p_data.get("goal", ""),
                status=p_data.get("status", "active"),
                color=p_data.get("color"),
                created_at=_dt(p_data.get("created_at")),
                updated_at=_dt(p_data.get("updated_at")),
            )
            session.add(proj)
            await session.flush()
            project_id_map[p_data["id"]] = proj.id
            stats["projects"] += 1

        # 3. Milestones
        for m_data in data.get("milestones", []):
            mapped_uid = user_id_map.get(m_data.get("user_id", 0))
            mapped_pid = project_id_map.get(m_data.get("project_id", 0))
            if mapped_uid is None or mapped_pid is None:
                continue
            ms = Milestone(
                user_id=mapped_uid,
                project_id=mapped_pid,
                title=m_data.get("title", ""),
                description=m_data.get("description"),
                status=m_data.get("status", "active"),
                order_index=m_data.get("order_index", 0),
                created_at=_dt(m_data.get("created_at")),
                updated_at=_dt(m_data.get("updated_at")),
            )
            session.add(ms)
            await session.flush()
            milestone_id_map[m_data["id"]] = ms.id
            stats["milestones"] += 1

        # 4a. Notes — first pass (no parent_id yet)
        notes_with_parents: list[tuple[int, int]] = []  # (new_note_id, old_parent_id)
        for n_data in data.get("notes", []):
            mapped_uid = user_id_map.get(n_data.get("user_id", 0))
            if mapped_uid is None:
                continue
            note = Note(
                user_id=mapped_uid,
                title=n_data.get("title", ""),
                body=n_data.get("body", ""),
                tags=n_data.get("tags", []),
                parent_id=None,
                project_id=project_id_map.get(n_data["project_id"]) if n_data.get("project_id") else None,
                milestone_id=milestone_id_map.get(n_data["milestone_id"]) if n_data.get("milestone_id") else None,
                status=n_data.get("status"),
                priority=n_data.get("priority"),
                due_date=_d(n_data.get("due_date")),
                created_at=_dt(n_data.get("created_at")),
                updated_at=_dt(n_data.get("updated_at")),
            )
            session.add(note)
            await session.flush()
            note_id_map[n_data["id"]] = note.id
            if n_data.get("parent_id"):
                notes_with_parents.append((note.id, n_data["parent_id"]))
            stats["notes"] += 1

        # 4b. Patch parent_id
        for new_note_id, old_parent_id in notes_with_parents:
            new_parent_id = note_id_map.get(old_parent_id)
            if new_parent_id:
                note_row = await session.get(Note, new_note_id)
                if note_row:
                    note_row.parent_id = new_parent_id

        # 5. TaskLogs
        for tl_data in data.get("task_logs", []):
            mapped_uid = user_id_map.get(tl_data.get("user_id", 0))
            mapped_tid = note_id_map.get(tl_data.get("task_id", 0))
            if mapped_uid is None or mapped_tid is None:
                continue
            tl = TaskLog(
                user_id=mapped_uid,
                task_id=mapped_tid,
                content=tl_data.get("content", ""),
                duration_minutes=tl_data.get("duration_minutes"),
                created_at=_dt(tl_data.get("created_at")),
                updated_at=_dt(tl_data.get("updated_at")),
            )
            session.add(tl)
            stats["task_logs"] += 1

        # 6. NoteDrafts
        for nd_data in data.get("note_drafts", []):
            mapped_uid = user_id_map.get(nd_data.get("user_id", 0))
            mapped_nid = note_id_map.get(nd_data.get("note_id", 0))
            if mapped_uid is None or mapped_nid is None:
                continue
            nd = NoteDraft(
                user_id=mapped_uid,
                note_id=mapped_nid,
                proposed_body=nd_data.get("proposed_body", ""),
                original_body=nd_data.get("original_body", ""),
                instruction=nd_data.get("instruction", ""),
                scope=nd_data.get("scope", "document"),
                created_at=_dt(nd_data.get("created_at")),
                updated_at=_dt(nd_data.get("updated_at")),
            )
            session.add(nd)
            stats["note_drafts"] += 1

        # 7. NoteVersions
        for nv_data in data.get("note_versions", []):
            mapped_uid = user_id_map.get(nv_data.get("user_id", 0))
            mapped_nid = note_id_map.get(nv_data.get("note_id", 0))
            if mapped_uid is None or mapped_nid is None:
                continue
            nv = NoteVersion(
                user_id=mapped_uid,
                note_id=mapped_nid,
                title=nv_data.get("title", ""),
                body=nv_data.get("body", ""),
                tags=nv_data.get("tags", []),
                pin_kind=nv_data.get("pin_kind"),
                pin_label=nv_data.get("pin_label"),
                created_at=_dt(nv_data.get("created_at")),
            )
            session.add(nv)
            stats["note_versions"] += 1

        # 8. Settings
        for s_data in data.get("settings", []):
            mapped_uid = user_id_map.get(s_data.get("user_id", 0))
            if mapped_uid is None:
                continue
            session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
            stats["settings"] += 1

        # 9. Rulebooks (v3)
        for rb_data in data.get("rulebooks", []):
            mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
            if mapped_uid is None:
                continue
            rb = Rulebook(
                owner_user_id=mapped_uid,
                title=rb_data.get("title", ""),
                description=rb_data.get("description", ""),
                always_on=rb_data.get("always_on", False),
                created_at=_dt(rb_data.get("created_at")),
                updated_at=_dt(rb_data.get("updated_at")),
            )
            session.add(rb)
            await session.flush()
            rulebook_id_map[rb_data["id"]] = rb.id
            stats["rulebooks"] += 1

        # 10. Topics (v3)
        for t_data in data.get("rulebook_topics", []):
            mapped_rbid = rulebook_id_map.get(t_data.get("rulebook_id", 0))
            if mapped_rbid is None:
                continue
            topic = RulebookTopic(
                rulebook_id=mapped_rbid,
                title=t_data.get("title", ""),
                description=t_data.get("description"),
                order_index=t_data.get("order_index", 0),
                created_at=_dt(t_data.get("created_at")),
                updated_at=_dt(t_data.get("updated_at")),
            )
            session.add(topic)
            await session.flush()
            topic_id_map[t_data["id"]] = topic.id
            stats["rulebook_topics"] += 1

        # 11. Rules (v3) — topic-rule (topic_id) XOR project-rule (project_id)
        for r_data in data.get("rules", []):
            mapped_topic = topic_id_map.get(r_data["topic_id"]) if r_data.get("topic_id") else None
            mapped_proj = project_id_map.get(r_data["project_id"]) if r_data.get("project_id") else None
            if mapped_topic is None and mapped_proj is None:
                continue  # orphaned — its parent didn't restore
            rule = Rule(
                topic_id=mapped_topic,
                project_id=mapped_proj,
                title=r_data.get("title", ""),
                statement=r_data.get("statement", ""),
                why=r_data.get("why") or None,
                how_to_apply=r_data.get("how_to_apply") or None,
                order_index=r_data.get("order_index", 0),
                created_at=_dt(r_data.get("created_at")),
                updated_at=_dt(r_data.get("updated_at")),
            )
            session.add(rule)
            await session.flush()
            rule_id_map[r_data["id"]] = rule.id
            stats["rules"] += 1

        # 12. Rulebook subscriptions (v3 join table)
        for sub in data.get("rulebook_subscriptions", []):
            mapped_pid = project_id_map.get(sub.get("project_id", 0))
            mapped_rbid = rulebook_id_map.get(sub.get("rulebook_id", 0))
            if mapped_pid is None or mapped_rbid is None:
                continue
            await session.execute(project_rulebook_subscriptions.insert().values(
                project_id=mapped_pid, rulebook_id=mapped_rbid,
            ))
            stats["rulebook_subscriptions"] += 1

        # 13. Rule suppressions (v3 join table)
        for sup in data.get("rule_suppressions", []):
            mapped_pid = project_id_map.get(sup.get("project_id", 0))
            mapped_rid = rule_id_map.get(sup.get("rule_id", 0))
            if mapped_pid is None or mapped_rid is None:
                continue
            await session.execute(project_rule_suppressions.insert().values(
                project_id=mapped_pid, rule_id=mapped_rid,
            ))
            stats["rule_suppressions"] += 1

        # 14. Topic suppressions (v3 join table)
        for sup in data.get("topic_suppressions", []):
            mapped_pid = project_id_map.get(sup.get("project_id", 0))
            mapped_tid = topic_id_map.get(sup.get("topic_id", 0))
            if mapped_pid is None or mapped_tid is None:
                continue
            await session.execute(project_topic_suppressions.insert().values(
                project_id=mapped_pid, topic_id=mapped_tid,
            ))
            stats["topic_suppressions"] += 1

        # --- v5 sections. Every one is data.get()-guarded, so a v2/v3/v4
        # payload restores without them rather than failing on an absent key.

        # 15. Systems
        system_id_map: dict[int, int] = {}
        for sy_data in data.get("systems", []):
            mapped_uid = user_id_map.get(sy_data.get("user_id", 0))
            mapped_pid = project_id_map.get(sy_data.get("project_id", 0))
            if mapped_uid is None or mapped_pid is None:
                continue
            system = System(
                user_id=mapped_uid, project_id=mapped_pid,
                name=sy_data.get("name", ""),
                description=sy_data.get("description"),
                color=sy_data.get("color"),
                status=sy_data.get("status", "active"),
                order_index=sy_data.get("order_index", 0),
            )
            session.add(system)
            await session.flush()
            system_id_map[sy_data["id"]] = system.id
            stats["systems"] += 1

        # 16. Record↔system links
        for rs in data.get("record_systems", []):
            mapped_nid = note_id_map.get(rs.get("note_id", 0))
            mapped_sid = system_id_map.get(rs.get("system_id", 0))
            if mapped_nid is None or mapped_sid is None:
                continue
            session.add(RecordSystem(note_id=mapped_nid, system_id=mapped_sid))
            stats["record_systems"] += 1

        # 17. Design systems. The export orders these parent-first, so a
        # parent's new id is always in the map by the time a child needs it —
        # no second pass, and a child whose parent is missing lands as a root
        # rather than failing the whole restore.
        design_system_id_map: dict[int, int] = {}
        for ds_data in data.get("design_systems", []):
            mapped_uid = user_id_map.get(ds_data.get("owner_user_id", 0))
            if mapped_uid is None:
                continue
            design = DesignSystem(
                owner_user_id=mapped_uid,
                title=ds_data.get("title", ""),
                description=ds_data.get("description"),
                guidance=ds_data.get("guidance"),
                parent_id=design_system_id_map.get(ds_data.get("parent_id") or 0),
            )
            session.add(design)
            await session.flush()
            design_system_id_map[ds_data["id"]] = design.id
            stats["design_systems"] += 1

        # 18. Design tokens
        for t_data in data.get("design_tokens", []):
            mapped_dsid = design_system_id_map.get(t_data.get("design_system_id", 0))
            if mapped_dsid is None:
                continue
            session.add(DesignToken(
                design_system_id=mapped_dsid,
                name=t_data.get("name", ""),
                value_by_mode=t_data.get("value_by_mode") or {},
                group_name=t_data.get("group_name"),
                purpose=t_data.get("purpose"),
                rationale=t_data.get("rationale"),
                supersedes=t_data.get("supersedes") or [],
                order_index=t_data.get("order_index", 0),
            ))
            stats["design_tokens"] += 1

        # 19. Usage events. Kept because pull-through is the evidence base for
        # whether recall works at all, and it is only ever accumulated — a
        # restore that dropped it would silently reset that measurement to zero
        # while everything still looked fine.
        for ev in data.get("note_usage_events", []):
            mapped_nid = note_id_map.get(ev.get("note_id", 0))
            if mapped_nid is None:
                continue
            session.add(NoteUsageEvent(
                user_id=user_id_map.get(ev.get("user_id") or 0),
                note_id=mapped_nid,
                event=ev.get("event", ""),
                source=ev.get("source", ""),
                created_at=_dt(ev.get("created_at")),
            ))
            stats["note_usage_events"] += 1

        # 20. Repo bindings — small, but losing them means every bound repo
        # quietly stops loading its project at session start.
        for rb_data in data.get("repo_bindings", []):
            mapped_uid = user_id_map.get(rb_data.get("user_id", 0))
            mapped_pid = project_id_map.get(rb_data.get("project_id", 0))
            if mapped_uid is None or mapped_pid is None:
                continue
            session.add(RepoBinding(
                user_id=mapped_uid, project_id=mapped_pid,
                repo_key=rb_data.get("repo_key", ""),
            ))
            stats["repo_bindings"] += 1

        await session.commit()

    logger.info("Restored v2/v3 backup: %s", stats)
    return stats