bvandeusen
5dcb738ce8
CI & Build / Python lint (push) Successful in 4s
CI & Build / integration (push) Successful in 13s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 48s
CI & Build / Build & push image (push) Successful in 36s
CI failed on 45c6b1c. `tests/test_services_backup.py` asserts every table is
either in `_BACKED_UP` or explicitly in `_NOT_INCLUDED`, and the new table was
in neither.
That guard exists for exactly this: a new table gets a model and a migration —
both of which fail loudly if wrong — and then silently never gets a backup
section. No error, no warning, and a restore that reports success. Its comment
says it was written to "stop the seventh" table slipping through. This was the
seventh.
Supersession claims are backed up rather than excluded because they are a
JUDGEMENT. Someone decided this note replaced that one, and nothing in either
note's text records the decision. Lose them and a restored corpus silently
returns to ranking stale material alongside current material — with no symptom
that says why.
Scoping differs by backup kind, deliberately. A whole-instance export takes
every row. A single-user export takes only claims where BOTH ends are that
user's notes: a claim spanning out to someone else's record cannot be restored
into a single-user import, since the far id is not in the map, so exporting it
would write a row that silently vanishes on the way back in.
Restore maps both ids and skips the row unless both resolve. A claim is about a
PAIR — half of one is not a weaker claim, it is a dangling row pointing at
whatever note holds that id next. Guarded by `data.get` like every post-v2
section, so v5 and older payloads restore cleanly without it.
BACKUP_VERSION 5 -> 6, and the version test moved with it. A payload section
added without moving the version produces backups that are structurally
different and indistinguishable by inspection.
Refs #278
1115 lines
45 KiB
Python
1115 lines
45 KiB
Python
import logging
|
|
from datetime import date, datetime, timezone
|
|
|
|
from sqlalchemy import or_, select
|
|
|
|
from scribe.models import async_session
|
|
from scribe.models.milestone import Milestone
|
|
from scribe.models.note import Note
|
|
from scribe.models.note_draft import NoteDraft
|
|
from scribe.models.note_supersession import NoteSupersession
|
|
from scribe.models.note_version import NoteVersion
|
|
from scribe.models.design_system import DesignSystem, DesignToken
|
|
from scribe.models.note_usage import NoteUsageEvent
|
|
from scribe.models.project import Project
|
|
from scribe.models.repo_binding import RepoBinding
|
|
from scribe.models.rulebook import (
|
|
Rule,
|
|
Rulebook,
|
|
RulebookTopic,
|
|
project_rule_suppressions,
|
|
project_rulebook_subscriptions,
|
|
project_topic_suppressions,
|
|
)
|
|
from scribe.models.setting import Setting
|
|
from scribe.models.system import RecordSystem, System
|
|
from scribe.models.task_log import TaskLog
|
|
from scribe.models.user import User
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Backup format version. v3 (2026-06) added rulebooks/topics/rules + their
|
|
# project subscription/suppression join tables. v4 (2026-07) dropped events
|
|
# when the calendar surface was retired — old v3 events are skipped on restore.
|
|
# v5 (2026-08) added the six tables that had accumulated outside the backup
|
|
# entirely (#2293), and the coverage guard that stops the seventh.
|
|
# v6 (2026-08) added note_supersessions — and the guard did stop the seventh:
|
|
# the table shipped without a backup section and the coverage test failed the
|
|
# build, which is the whole reason that list was written.
|
|
# Bump when the serialized schema changes.
|
|
BACKUP_VERSION = 6
|
|
|
|
# Every table this backup carries, by its REAL name. Paired with _NOT_INCLUDED
|
|
# below, these two lists must together account for the entire schema — which is
|
|
# what tests/test_services_backup.py asserts against Base.metadata.
|
|
#
|
|
# The point is the ABSENCE case. A new table gets a model and a migration, both
|
|
# of which fail loudly if wrong, and then silently never gets a backup section:
|
|
# no error, no warning, and a restore that reports success. Naming the coverage
|
|
# explicitly turns "someone forgot" into a failing test (#2293).
|
|
_BACKED_UP = [
|
|
"users", "projects", "milestones", "notes", "task_logs", "note_drafts",
|
|
"note_versions", "settings", "rulebooks", "rulebook_topics", "rules",
|
|
"project_rulebook_subscriptions", "project_rule_suppressions",
|
|
"project_topic_suppressions",
|
|
# v5 (2026-08): the five-year gap this list was written to stop.
|
|
"systems", "record_systems", "design_systems", "design_tokens",
|
|
"note_usage_events", "repo_bindings", "note_supersessions",
|
|
]
|
|
|
|
# Tables intentionally NOT in the backup, surfaced in the payload so the gap is
|
|
# explicit rather than silent. ACL (groups/shares) is a coherent follow-up;
|
|
# note_embeddings are derived (regenerated from note bodies); api_keys are
|
|
# sensitive credentials; retrieval_logs is observational telemetry that nothing
|
|
# reads for correctness and that grows per query; the rest are
|
|
# transient/operational.
|
|
#
|
|
# REAL table names, deliberately. This list used to read "embeddings",
|
|
# "invitations", "password_resets" — none of which are tables — so it looked
|
|
# like coverage while naming nothing the schema could confirm.
|
|
_NOT_INCLUDED = [
|
|
"groups", "group_memberships", "project_shares", "note_shares",
|
|
"api_keys", "note_embeddings", "app_logs", "notifications",
|
|
"invitation_tokens", "password_reset_tokens", "user_profiles",
|
|
"retrieval_logs",
|
|
]
|
|
|
|
|
|
def _dt(val: str | None) -> datetime:
|
|
return datetime.fromisoformat(val) if val else datetime.now(timezone.utc)
|
|
|
|
|
|
def _d(val: str | None) -> date | None:
|
|
return date.fromisoformat(val) if val else None
|
|
|
|
|
|
def _subscription_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "rulebook_id": r.rulebook_id} for r in rows]
|
|
|
|
|
|
def _rule_suppression_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "rule_id": r.rule_id} for r in rows]
|
|
|
|
|
|
def _topic_suppression_rows(rows) -> list[dict]:
|
|
return [{"project_id": r.project_id, "topic_id": r.topic_id} for r in rows]
|
|
|
|
|
|
# The v5 sections. Pure row-builders like the join-table helpers above, for the
|
|
# same reason: CI has no database, so a serialiser that is a plain function is
|
|
# one that can actually be tested.
|
|
|
|
def _system_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "user_id": r.user_id, "project_id": r.project_id,
|
|
"name": r.name, "description": r.description, "color": r.color,
|
|
"status": r.status, "order_index": r.order_index,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _record_system_rows(rows) -> list[dict]:
|
|
return [{"note_id": r.note_id, "system_id": r.system_id} for r in rows]
|
|
|
|
|
|
def _note_supersession_rows(rows) -> list[dict]:
|
|
"""Which record has overtaken which. Carried because it is a JUDGEMENT —
|
|
someone decided this note replaced that one, and nothing in either note's
|
|
text records the decision. Lose it and the corpus silently reverts to
|
|
ranking stale material alongside current material."""
|
|
return [
|
|
{"superseder_id": r.superseder_id, "superseded_id": r.superseded_id}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _design_system_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "owner_user_id": r.owner_user_id, "title": r.title,
|
|
"description": r.description, "guidance": r.guidance,
|
|
"parent_id": r.parent_id,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _design_token_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"id": r.id, "design_system_id": r.design_system_id, "name": r.name,
|
|
"value_by_mode": r.value_by_mode or {},
|
|
"group_name": r.group_name, "purpose": r.purpose,
|
|
"rationale": r.rationale, "supersedes": r.supersedes or [],
|
|
"order_index": r.order_index,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _usage_event_rows(rows) -> list[dict]:
|
|
return [
|
|
{
|
|
"user_id": r.user_id, "note_id": r.note_id, "event": r.event,
|
|
"source": r.source,
|
|
"created_at": r.created_at.isoformat() if r.created_at else None,
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def _repo_binding_rows(rows) -> list[dict]:
|
|
return [
|
|
{"user_id": r.user_id, "project_id": r.project_id, "repo_key": r.repo_key}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Export
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def export_full_backup() -> dict:
|
|
"""Export all data as a version-5 JSON backup."""
|
|
async with async_session() as session:
|
|
users = (await session.execute(select(User))).scalars().all()
|
|
projects = (await session.execute(select(Project))).scalars().all()
|
|
milestones = (await session.execute(select(Milestone))).scalars().all()
|
|
notes = (await session.execute(select(Note))).scalars().all()
|
|
task_logs = (await session.execute(select(TaskLog))).scalars().all()
|
|
note_drafts = (await session.execute(select(NoteDraft))).scalars().all()
|
|
note_versions = (await session.execute(
|
|
select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id)
|
|
)).scalars().all()
|
|
settings = (await session.execute(select(Setting))).scalars().all()
|
|
systems = (await session.execute(select(System))).scalars().all()
|
|
record_systems = (await session.execute(select(RecordSystem))).scalars().all()
|
|
supersessions = (
|
|
await session.execute(select(NoteSupersession))
|
|
).scalars().all()
|
|
# Parent-first, so a restore can resolve parent_id as it goes rather
|
|
# than needing a second pass — the self-FK is the only ordering
|
|
# constraint in this payload.
|
|
design_systems = (await session.execute(
|
|
select(DesignSystem).order_by(DesignSystem.parent_id.nullsfirst(),
|
|
DesignSystem.id)
|
|
)).scalars().all()
|
|
design_tokens = (await session.execute(select(DesignToken))).scalars().all()
|
|
usage_events = (await session.execute(select(NoteUsageEvent))).scalars().all()
|
|
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
|
|
rulebooks = (await session.execute(select(Rulebook))).scalars().all()
|
|
topics = (await session.execute(select(RulebookTopic))).scalars().all()
|
|
rules = (await session.execute(select(Rule))).scalars().all()
|
|
subscriptions = (await session.execute(
|
|
select(project_rulebook_subscriptions)
|
|
)).all()
|
|
rule_suppressions = (await session.execute(
|
|
select(project_rule_suppressions)
|
|
)).all()
|
|
topic_suppressions = (await session.execute(
|
|
select(project_topic_suppressions)
|
|
)).all()
|
|
|
|
return {
|
|
"version": BACKUP_VERSION,
|
|
"scope": "full",
|
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
|
"_security_notice": (
|
|
"This backup contains hashed passwords. "
|
|
"Store it securely and restrict access."
|
|
),
|
|
"_not_included": _NOT_INCLUDED,
|
|
"users": [
|
|
{
|
|
"id": u.id,
|
|
"username": u.username,
|
|
"email": u.email,
|
|
"password_hash": u.password_hash,
|
|
"oauth_sub": u.oauth_sub,
|
|
"role": u.role,
|
|
"session_version": u.session_version,
|
|
"created_at": u.created_at.isoformat(),
|
|
}
|
|
for u in users
|
|
],
|
|
"projects": [
|
|
{
|
|
"id": p.id,
|
|
"user_id": p.user_id,
|
|
"title": p.title,
|
|
"description": p.description,
|
|
"goal": p.goal,
|
|
"status": p.status,
|
|
"color": p.color,
|
|
"created_at": p.created_at.isoformat(),
|
|
"updated_at": p.updated_at.isoformat(),
|
|
}
|
|
for p in projects
|
|
],
|
|
"milestones": [
|
|
{
|
|
"id": m.id,
|
|
"user_id": m.user_id,
|
|
"project_id": m.project_id,
|
|
"title": m.title,
|
|
"description": m.description,
|
|
"status": m.status,
|
|
"order_index": m.order_index,
|
|
"created_at": m.created_at.isoformat(),
|
|
"updated_at": m.updated_at.isoformat(),
|
|
}
|
|
for m in milestones
|
|
],
|
|
"notes": [
|
|
{
|
|
"id": n.id,
|
|
"user_id": n.user_id,
|
|
"title": n.title,
|
|
"body": n.body,
|
|
"tags": n.tags or [],
|
|
"parent_id": n.parent_id,
|
|
"project_id": n.project_id,
|
|
"milestone_id": n.milestone_id,
|
|
"status": n.status,
|
|
"priority": n.priority,
|
|
"due_date": n.due_date.isoformat() if n.due_date else None,
|
|
"created_at": n.created_at.isoformat(),
|
|
"updated_at": n.updated_at.isoformat(),
|
|
}
|
|
for n in notes
|
|
],
|
|
"task_logs": [
|
|
{
|
|
"id": tl.id,
|
|
"user_id": tl.user_id,
|
|
"task_id": tl.task_id,
|
|
"content": tl.content,
|
|
"duration_minutes": tl.duration_minutes,
|
|
"created_at": tl.created_at.isoformat(),
|
|
"updated_at": tl.updated_at.isoformat(),
|
|
}
|
|
for tl in task_logs
|
|
],
|
|
"note_drafts": [
|
|
{
|
|
"id": nd.id,
|
|
"user_id": nd.user_id,
|
|
"note_id": nd.note_id,
|
|
"proposed_body": nd.proposed_body,
|
|
"original_body": nd.original_body,
|
|
"instruction": nd.instruction,
|
|
"scope": nd.scope,
|
|
"created_at": nd.created_at.isoformat(),
|
|
"updated_at": nd.updated_at.isoformat(),
|
|
}
|
|
for nd in note_drafts
|
|
],
|
|
"note_versions": [
|
|
{
|
|
"id": nv.id,
|
|
"user_id": nv.user_id,
|
|
"note_id": nv.note_id,
|
|
"title": nv.title,
|
|
"body": nv.body,
|
|
"tags": nv.tags or [],
|
|
"pin_kind": nv.pin_kind,
|
|
"pin_label": nv.pin_label,
|
|
"created_at": nv.created_at.isoformat(),
|
|
}
|
|
for nv in note_versions
|
|
],
|
|
"settings": [
|
|
{"user_id": s.user_id, "key": s.key, "value": s.value}
|
|
for s in settings
|
|
],
|
|
"rulebooks": [
|
|
{
|
|
"id": rb.id,
|
|
"owner_user_id": rb.owner_user_id,
|
|
"title": rb.title,
|
|
"description": rb.description,
|
|
"always_on": rb.always_on,
|
|
"created_at": rb.created_at.isoformat(),
|
|
"updated_at": rb.updated_at.isoformat(),
|
|
}
|
|
for rb in rulebooks
|
|
],
|
|
"rulebook_topics": [
|
|
{
|
|
"id": t.id,
|
|
"rulebook_id": t.rulebook_id,
|
|
"title": t.title,
|
|
"description": t.description,
|
|
"order_index": t.order_index,
|
|
"created_at": t.created_at.isoformat(),
|
|
"updated_at": t.updated_at.isoformat(),
|
|
}
|
|
for t in topics
|
|
],
|
|
"rules": [
|
|
{
|
|
"id": r.id,
|
|
"topic_id": r.topic_id,
|
|
"project_id": r.project_id,
|
|
"title": r.title,
|
|
"statement": r.statement,
|
|
"why": r.why,
|
|
"how_to_apply": r.how_to_apply,
|
|
"order_index": r.order_index,
|
|
"created_at": r.created_at.isoformat(),
|
|
"updated_at": r.updated_at.isoformat(),
|
|
}
|
|
for r in rules
|
|
],
|
|
"rulebook_subscriptions": _subscription_rows(subscriptions),
|
|
"rule_suppressions": _rule_suppression_rows(rule_suppressions),
|
|
"topic_suppressions": _topic_suppression_rows(topic_suppressions),
|
|
"systems": _system_rows(systems),
|
|
"record_systems": _record_system_rows(record_systems),
|
|
"design_systems": _design_system_rows(design_systems),
|
|
"design_tokens": _design_token_rows(design_tokens),
|
|
"note_usage_events": _usage_event_rows(usage_events),
|
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
|
"note_supersessions": _note_supersession_rows(supersessions),
|
|
}
|
|
|
|
|
|
async def export_user_backup(user_id: int) -> dict:
|
|
"""Export a single user's data as a version-5 JSON backup."""
|
|
async with async_session() as session:
|
|
user = await session.get(User, user_id)
|
|
projects = (await session.execute(
|
|
select(Project).where(Project.user_id == user_id)
|
|
)).scalars().all()
|
|
project_ids = [p.id for p in projects]
|
|
milestones = (await session.execute(
|
|
select(Milestone).where(Milestone.user_id == user_id)
|
|
)).scalars().all()
|
|
notes = (await session.execute(
|
|
select(Note).where(Note.user_id == user_id)
|
|
)).scalars().all()
|
|
task_logs = (await session.execute(
|
|
select(TaskLog).where(TaskLog.user_id == user_id)
|
|
)).scalars().all()
|
|
note_drafts = (await session.execute(
|
|
select(NoteDraft).where(NoteDraft.user_id == user_id)
|
|
)).scalars().all()
|
|
note_versions = (await session.execute(
|
|
select(NoteVersion).where(NoteVersion.user_id == user_id)
|
|
.order_by(NoteVersion.note_id, NoteVersion.id)
|
|
)).scalars().all()
|
|
settings = (await session.execute(
|
|
select(Setting).where(Setting.user_id == user_id)
|
|
)).scalars().all()
|
|
systems = (await session.execute(
|
|
select(System).where(System.user_id == user_id)
|
|
)).scalars().all()
|
|
system_ids = [sy.id for sy in systems]
|
|
note_ids = [n.id for n in notes]
|
|
# Scoped by the user's SYSTEMS, not their notes: a shared note carrying
|
|
# this user's system tag belongs in their backup, and a note of theirs
|
|
# tagged with someone else's system does not — that row is the other
|
|
# user's to keep.
|
|
record_systems = (await session.execute(
|
|
select(RecordSystem).where(RecordSystem.system_id.in_(system_ids))
|
|
)).scalars().all() if system_ids else []
|
|
# BOTH ends must be this user's notes. A claim spanning out to someone
|
|
# else's record cannot be restored into a single-user import — the far
|
|
# id would not be in the map — so carrying it would export a row that
|
|
# silently vanishes on the way back in. Whole-instance backups have no
|
|
# such problem and take every row.
|
|
supersessions = (await session.execute(
|
|
select(NoteSupersession).where(
|
|
NoteSupersession.superseder_id.in_(note_ids),
|
|
NoteSupersession.superseded_id.in_(note_ids),
|
|
)
|
|
)).scalars().all() if note_ids else []
|
|
design_systems = (await session.execute(
|
|
select(DesignSystem).where(DesignSystem.owner_user_id == user_id)
|
|
.order_by(DesignSystem.parent_id.nullsfirst(), DesignSystem.id)
|
|
)).scalars().all()
|
|
ds_ids = [d.id for d in design_systems]
|
|
design_tokens = (await session.execute(
|
|
select(DesignToken).where(DesignToken.design_system_id.in_(ds_ids))
|
|
)).scalars().all() if ds_ids else []
|
|
usage_events = (await session.execute(
|
|
select(NoteUsageEvent).where(NoteUsageEvent.note_id.in_(note_ids))
|
|
)).scalars().all() if note_ids else []
|
|
repo_bindings = (await session.execute(
|
|
select(RepoBinding).where(RepoBinding.user_id == user_id)
|
|
)).scalars().all()
|
|
rulebooks = (await session.execute(
|
|
select(Rulebook).where(Rulebook.owner_user_id == user_id)
|
|
)).scalars().all()
|
|
rulebook_ids = [rb.id for rb in rulebooks]
|
|
topics = (await session.execute(
|
|
select(RulebookTopic).where(RulebookTopic.rulebook_id.in_(rulebook_ids))
|
|
)).scalars().all() if rulebook_ids else []
|
|
topic_ids = [t.id for t in topics]
|
|
# Rules owned by the user = rules in the user's topics OR project-rules
|
|
# on the user's projects.
|
|
rule_filters = []
|
|
if topic_ids:
|
|
rule_filters.append(Rule.topic_id.in_(topic_ids))
|
|
if project_ids:
|
|
rule_filters.append(Rule.project_id.in_(project_ids))
|
|
rules = (await session.execute(
|
|
select(Rule).where(or_(*rule_filters))
|
|
)).scalars().all() if rule_filters else []
|
|
if project_ids:
|
|
subscriptions = (await session.execute(
|
|
select(project_rulebook_subscriptions).where(
|
|
project_rulebook_subscriptions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
rule_suppressions = (await session.execute(
|
|
select(project_rule_suppressions).where(
|
|
project_rule_suppressions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
topic_suppressions = (await session.execute(
|
|
select(project_topic_suppressions).where(
|
|
project_topic_suppressions.c.project_id.in_(project_ids)
|
|
)
|
|
)).all()
|
|
else:
|
|
subscriptions = rule_suppressions = topic_suppressions = []
|
|
|
|
return {
|
|
"version": BACKUP_VERSION,
|
|
"scope": "user",
|
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
|
"_not_included": _NOT_INCLUDED,
|
|
"user": {
|
|
"id": user.id,
|
|
"username": user.username,
|
|
"email": user.email,
|
|
"role": user.role,
|
|
"created_at": user.created_at.isoformat(),
|
|
} if user else None,
|
|
"projects": [
|
|
{
|
|
"id": p.id,
|
|
"user_id": p.user_id,
|
|
"title": p.title,
|
|
"description": p.description,
|
|
"goal": p.goal,
|
|
"status": p.status,
|
|
"color": p.color,
|
|
"created_at": p.created_at.isoformat(),
|
|
"updated_at": p.updated_at.isoformat(),
|
|
}
|
|
for p in projects
|
|
],
|
|
"milestones": [
|
|
{
|
|
"id": m.id,
|
|
"user_id": m.user_id,
|
|
"project_id": m.project_id,
|
|
"title": m.title,
|
|
"description": m.description,
|
|
"status": m.status,
|
|
"order_index": m.order_index,
|
|
"created_at": m.created_at.isoformat(),
|
|
"updated_at": m.updated_at.isoformat(),
|
|
}
|
|
for m in milestones
|
|
],
|
|
"notes": [
|
|
{
|
|
"id": n.id,
|
|
"user_id": n.user_id,
|
|
"title": n.title,
|
|
"body": n.body,
|
|
"tags": n.tags or [],
|
|
"parent_id": n.parent_id,
|
|
"project_id": n.project_id,
|
|
"milestone_id": n.milestone_id,
|
|
"status": n.status,
|
|
"priority": n.priority,
|
|
"due_date": n.due_date.isoformat() if n.due_date else None,
|
|
"created_at": n.created_at.isoformat(),
|
|
"updated_at": n.updated_at.isoformat(),
|
|
}
|
|
for n in notes
|
|
],
|
|
"task_logs": [
|
|
{
|
|
"id": tl.id,
|
|
"user_id": tl.user_id,
|
|
"task_id": tl.task_id,
|
|
"content": tl.content,
|
|
"duration_minutes": tl.duration_minutes,
|
|
"created_at": tl.created_at.isoformat(),
|
|
"updated_at": tl.updated_at.isoformat(),
|
|
}
|
|
for tl in task_logs
|
|
],
|
|
"note_drafts": [
|
|
{
|
|
"id": nd.id,
|
|
"user_id": nd.user_id,
|
|
"note_id": nd.note_id,
|
|
"proposed_body": nd.proposed_body,
|
|
"original_body": nd.original_body,
|
|
"instruction": nd.instruction,
|
|
"scope": nd.scope,
|
|
"created_at": nd.created_at.isoformat(),
|
|
"updated_at": nd.updated_at.isoformat(),
|
|
}
|
|
for nd in note_drafts
|
|
],
|
|
"note_versions": [
|
|
{
|
|
"id": nv.id,
|
|
"user_id": nv.user_id,
|
|
"note_id": nv.note_id,
|
|
"title": nv.title,
|
|
"body": nv.body,
|
|
"tags": nv.tags or [],
|
|
"pin_kind": nv.pin_kind,
|
|
"pin_label": nv.pin_label,
|
|
"created_at": nv.created_at.isoformat(),
|
|
}
|
|
for nv in note_versions
|
|
],
|
|
"settings": [
|
|
{"user_id": s.user_id, "key": s.key, "value": s.value}
|
|
for s in settings
|
|
],
|
|
"rulebooks": [
|
|
{
|
|
"id": rb.id,
|
|
"owner_user_id": rb.owner_user_id,
|
|
"title": rb.title,
|
|
"description": rb.description,
|
|
"always_on": rb.always_on,
|
|
"created_at": rb.created_at.isoformat(),
|
|
"updated_at": rb.updated_at.isoformat(),
|
|
}
|
|
for rb in rulebooks
|
|
],
|
|
"rulebook_topics": [
|
|
{
|
|
"id": t.id,
|
|
"rulebook_id": t.rulebook_id,
|
|
"title": t.title,
|
|
"description": t.description,
|
|
"order_index": t.order_index,
|
|
"created_at": t.created_at.isoformat(),
|
|
"updated_at": t.updated_at.isoformat(),
|
|
}
|
|
for t in topics
|
|
],
|
|
"rules": [
|
|
{
|
|
"id": r.id,
|
|
"topic_id": r.topic_id,
|
|
"project_id": r.project_id,
|
|
"title": r.title,
|
|
"statement": r.statement,
|
|
"why": r.why,
|
|
"how_to_apply": r.how_to_apply,
|
|
"order_index": r.order_index,
|
|
"created_at": r.created_at.isoformat(),
|
|
"updated_at": r.updated_at.isoformat(),
|
|
}
|
|
for r in rules
|
|
],
|
|
"rulebook_subscriptions": _subscription_rows(subscriptions),
|
|
"rule_suppressions": _rule_suppression_rows(rule_suppressions),
|
|
"topic_suppressions": _topic_suppression_rows(topic_suppressions),
|
|
"systems": _system_rows(systems),
|
|
"record_systems": _record_system_rows(record_systems),
|
|
"design_systems": _design_system_rows(design_systems),
|
|
"design_tokens": _design_token_rows(design_tokens),
|
|
"note_usage_events": _usage_event_rows(usage_events),
|
|
"repo_bindings": _repo_binding_rows(repo_bindings),
|
|
"note_supersessions": _note_supersession_rows(supersessions),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Restore
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def restore_full_backup(data: dict) -> dict:
|
|
"""Restore from backup JSON. Dispatches by version."""
|
|
version = data.get("version", 1)
|
|
if version == 1:
|
|
return await _restore_v1(data)
|
|
# v2 and v3 share one path; v3-only sections are guarded by data.get so a
|
|
# v2 payload (without them) restores cleanly.
|
|
return await _restore_v2(data)
|
|
|
|
|
|
async def _restore_v1(data: dict) -> dict:
|
|
"""Restore legacy v1 backup (original format).
|
|
|
|
Pre-pivot v1 backups included conversations + messages; those are
|
|
skipped during restore now that the chat subsystem is gone.
|
|
"""
|
|
stats = {"users": 0, "notes": 0, "settings": 0}
|
|
|
|
async with async_session() as session:
|
|
user_id_map: dict[int, int] = {}
|
|
for u_data in data.get("users", []):
|
|
old_id = u_data["id"]
|
|
user = User(
|
|
username=u_data["username"],
|
|
email=u_data.get("email"),
|
|
password_hash=u_data["password_hash"],
|
|
role=u_data.get("role", "user"),
|
|
created_at=_dt(u_data.get("created_at")),
|
|
)
|
|
session.add(user)
|
|
await session.flush()
|
|
user_id_map[old_id] = user.id
|
|
stats["users"] += 1
|
|
|
|
note_id_map: dict[int, int] = {}
|
|
for n_data in data.get("notes", []):
|
|
old_id = n_data.get("id")
|
|
mapped_user_id = user_id_map.get(n_data.get("user_id", 0))
|
|
if mapped_user_id is None:
|
|
continue
|
|
note = Note(
|
|
user_id=mapped_user_id,
|
|
title=n_data.get("title", ""),
|
|
body=n_data.get("body", ""),
|
|
tags=n_data.get("tags", []),
|
|
parent_id=None, # patched below
|
|
status=n_data.get("status"),
|
|
priority=n_data.get("priority"),
|
|
due_date=_d(n_data.get("due_date")),
|
|
created_at=_dt(n_data.get("created_at")),
|
|
updated_at=_dt(n_data.get("updated_at")),
|
|
)
|
|
session.add(note)
|
|
await session.flush()
|
|
if old_id is not None:
|
|
note_id_map[old_id] = note.id
|
|
stats["notes"] += 1
|
|
|
|
# Patch parent_id now that all notes have new IDs
|
|
for n_data in data.get("notes", []):
|
|
old_id = n_data.get("id")
|
|
old_parent = n_data.get("parent_id")
|
|
if old_id and old_parent and old_id in note_id_map and old_parent in note_id_map:
|
|
note_row = await session.get(Note, note_id_map[old_id])
|
|
if note_row:
|
|
note_row.parent_id = note_id_map[old_parent]
|
|
|
|
for s_data in data.get("settings", []):
|
|
mapped_user_id = user_id_map.get(s_data.get("user_id", 0))
|
|
if mapped_user_id is None:
|
|
continue
|
|
session.add(Setting(user_id=mapped_user_id, key=s_data["key"], value=s_data.get("value", "")))
|
|
stats["settings"] += 1
|
|
|
|
await session.commit()
|
|
|
|
logger.info("Restored v1 backup: %s", stats)
|
|
return stats
|
|
|
|
|
|
async def _restore_v2(data: dict) -> dict:
|
|
"""Restore v2/v3 backup with full FK re-mapping.
|
|
|
|
Conversations, push subscriptions, and (as of v4) events in older backups
|
|
are silently skipped — those subsystems were removed. v3+ sections
|
|
(rulebooks/topics/rules/join-tables) are guarded by data.get so a v2
|
|
payload restores without them.
|
|
"""
|
|
stats: dict[str, int] = {
|
|
"users": 0, "projects": 0, "milestones": 0, "notes": 0,
|
|
"task_logs": 0, "note_drafts": 0, "note_versions": 0,
|
|
"settings": 0, "rulebooks": 0, "rulebook_topics": 0, "rules": 0,
|
|
"rulebook_subscriptions": 0, "rule_suppressions": 0,
|
|
"topic_suppressions": 0,
|
|
"systems": 0, "record_systems": 0, "design_systems": 0,
|
|
"design_tokens": 0, "note_usage_events": 0, "repo_bindings": 0,
|
|
"note_supersessions": 0,
|
|
}
|
|
|
|
async with async_session() as session:
|
|
user_id_map: dict[int, int] = {}
|
|
project_id_map: dict[int, int] = {}
|
|
milestone_id_map: dict[int, int] = {}
|
|
note_id_map: dict[int, int] = {}
|
|
rulebook_id_map: dict[int, int] = {}
|
|
topic_id_map: dict[int, int] = {}
|
|
rule_id_map: dict[int, int] = {}
|
|
|
|
# 1. Users
|
|
for u_data in data.get("users", []):
|
|
old_id = u_data["id"]
|
|
user = User(
|
|
username=u_data["username"],
|
|
email=u_data.get("email"),
|
|
password_hash=u_data.get("password_hash"),
|
|
oauth_sub=u_data.get("oauth_sub"),
|
|
role=u_data.get("role", "user"),
|
|
session_version=u_data.get("session_version", 1),
|
|
created_at=_dt(u_data.get("created_at")),
|
|
)
|
|
session.add(user)
|
|
await session.flush()
|
|
user_id_map[old_id] = user.id
|
|
stats["users"] += 1
|
|
|
|
# 2. Projects
|
|
for p_data in data.get("projects", []):
|
|
mapped_uid = user_id_map.get(p_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
proj = Project(
|
|
user_id=mapped_uid,
|
|
title=p_data.get("title", ""),
|
|
description=p_data.get("description", ""),
|
|
goal=p_data.get("goal", ""),
|
|
status=p_data.get("status", "active"),
|
|
color=p_data.get("color"),
|
|
created_at=_dt(p_data.get("created_at")),
|
|
updated_at=_dt(p_data.get("updated_at")),
|
|
)
|
|
session.add(proj)
|
|
await session.flush()
|
|
project_id_map[p_data["id"]] = proj.id
|
|
stats["projects"] += 1
|
|
|
|
# 3. Milestones
|
|
for m_data in data.get("milestones", []):
|
|
mapped_uid = user_id_map.get(m_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(m_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
ms = Milestone(
|
|
user_id=mapped_uid,
|
|
project_id=mapped_pid,
|
|
title=m_data.get("title", ""),
|
|
description=m_data.get("description"),
|
|
status=m_data.get("status", "active"),
|
|
order_index=m_data.get("order_index", 0),
|
|
created_at=_dt(m_data.get("created_at")),
|
|
updated_at=_dt(m_data.get("updated_at")),
|
|
)
|
|
session.add(ms)
|
|
await session.flush()
|
|
milestone_id_map[m_data["id"]] = ms.id
|
|
stats["milestones"] += 1
|
|
|
|
# 4a. Notes — first pass (no parent_id yet)
|
|
notes_with_parents: list[tuple[int, int]] = [] # (new_note_id, old_parent_id)
|
|
for n_data in data.get("notes", []):
|
|
mapped_uid = user_id_map.get(n_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
note = Note(
|
|
user_id=mapped_uid,
|
|
title=n_data.get("title", ""),
|
|
body=n_data.get("body", ""),
|
|
tags=n_data.get("tags", []),
|
|
parent_id=None,
|
|
project_id=project_id_map.get(n_data["project_id"]) if n_data.get("project_id") else None,
|
|
milestone_id=milestone_id_map.get(n_data["milestone_id"]) if n_data.get("milestone_id") else None,
|
|
status=n_data.get("status"),
|
|
priority=n_data.get("priority"),
|
|
due_date=_d(n_data.get("due_date")),
|
|
created_at=_dt(n_data.get("created_at")),
|
|
updated_at=_dt(n_data.get("updated_at")),
|
|
)
|
|
session.add(note)
|
|
await session.flush()
|
|
note_id_map[n_data["id"]] = note.id
|
|
if n_data.get("parent_id"):
|
|
notes_with_parents.append((note.id, n_data["parent_id"]))
|
|
stats["notes"] += 1
|
|
|
|
# 4b. Patch parent_id
|
|
for new_note_id, old_parent_id in notes_with_parents:
|
|
new_parent_id = note_id_map.get(old_parent_id)
|
|
if new_parent_id:
|
|
note_row = await session.get(Note, new_note_id)
|
|
if note_row:
|
|
note_row.parent_id = new_parent_id
|
|
|
|
# 5. TaskLogs
|
|
for tl_data in data.get("task_logs", []):
|
|
mapped_uid = user_id_map.get(tl_data.get("user_id", 0))
|
|
mapped_tid = note_id_map.get(tl_data.get("task_id", 0))
|
|
if mapped_uid is None or mapped_tid is None:
|
|
continue
|
|
tl = TaskLog(
|
|
user_id=mapped_uid,
|
|
task_id=mapped_tid,
|
|
content=tl_data.get("content", ""),
|
|
duration_minutes=tl_data.get("duration_minutes"),
|
|
created_at=_dt(tl_data.get("created_at")),
|
|
updated_at=_dt(tl_data.get("updated_at")),
|
|
)
|
|
session.add(tl)
|
|
stats["task_logs"] += 1
|
|
|
|
# 6. NoteDrafts
|
|
for nd_data in data.get("note_drafts", []):
|
|
mapped_uid = user_id_map.get(nd_data.get("user_id", 0))
|
|
mapped_nid = note_id_map.get(nd_data.get("note_id", 0))
|
|
if mapped_uid is None or mapped_nid is None:
|
|
continue
|
|
nd = NoteDraft(
|
|
user_id=mapped_uid,
|
|
note_id=mapped_nid,
|
|
proposed_body=nd_data.get("proposed_body", ""),
|
|
original_body=nd_data.get("original_body", ""),
|
|
instruction=nd_data.get("instruction", ""),
|
|
scope=nd_data.get("scope", "document"),
|
|
created_at=_dt(nd_data.get("created_at")),
|
|
updated_at=_dt(nd_data.get("updated_at")),
|
|
)
|
|
session.add(nd)
|
|
stats["note_drafts"] += 1
|
|
|
|
# 7. NoteVersions
|
|
for nv_data in data.get("note_versions", []):
|
|
mapped_uid = user_id_map.get(nv_data.get("user_id", 0))
|
|
mapped_nid = note_id_map.get(nv_data.get("note_id", 0))
|
|
if mapped_uid is None or mapped_nid is None:
|
|
continue
|
|
nv = NoteVersion(
|
|
user_id=mapped_uid,
|
|
note_id=mapped_nid,
|
|
title=nv_data.get("title", ""),
|
|
body=nv_data.get("body", ""),
|
|
tags=nv_data.get("tags", []),
|
|
pin_kind=nv_data.get("pin_kind"),
|
|
pin_label=nv_data.get("pin_label"),
|
|
created_at=_dt(nv_data.get("created_at")),
|
|
)
|
|
session.add(nv)
|
|
stats["note_versions"] += 1
|
|
|
|
# 8. Settings
|
|
for s_data in data.get("settings", []):
|
|
mapped_uid = user_id_map.get(s_data.get("user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
session.add(Setting(user_id=mapped_uid, key=s_data["key"], value=s_data.get("value", "")))
|
|
stats["settings"] += 1
|
|
|
|
# 9. Rulebooks (v3)
|
|
for rb_data in data.get("rulebooks", []):
|
|
mapped_uid = user_id_map.get(rb_data.get("owner_user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
rb = Rulebook(
|
|
owner_user_id=mapped_uid,
|
|
title=rb_data.get("title", ""),
|
|
description=rb_data.get("description", ""),
|
|
always_on=rb_data.get("always_on", False),
|
|
created_at=_dt(rb_data.get("created_at")),
|
|
updated_at=_dt(rb_data.get("updated_at")),
|
|
)
|
|
session.add(rb)
|
|
await session.flush()
|
|
rulebook_id_map[rb_data["id"]] = rb.id
|
|
stats["rulebooks"] += 1
|
|
|
|
# 10. Topics (v3)
|
|
for t_data in data.get("rulebook_topics", []):
|
|
mapped_rbid = rulebook_id_map.get(t_data.get("rulebook_id", 0))
|
|
if mapped_rbid is None:
|
|
continue
|
|
topic = RulebookTopic(
|
|
rulebook_id=mapped_rbid,
|
|
title=t_data.get("title", ""),
|
|
description=t_data.get("description"),
|
|
order_index=t_data.get("order_index", 0),
|
|
created_at=_dt(t_data.get("created_at")),
|
|
updated_at=_dt(t_data.get("updated_at")),
|
|
)
|
|
session.add(topic)
|
|
await session.flush()
|
|
topic_id_map[t_data["id"]] = topic.id
|
|
stats["rulebook_topics"] += 1
|
|
|
|
# 11. Rules (v3) — topic-rule (topic_id) XOR project-rule (project_id)
|
|
for r_data in data.get("rules", []):
|
|
mapped_topic = topic_id_map.get(r_data["topic_id"]) if r_data.get("topic_id") else None
|
|
mapped_proj = project_id_map.get(r_data["project_id"]) if r_data.get("project_id") else None
|
|
if mapped_topic is None and mapped_proj is None:
|
|
continue # orphaned — its parent didn't restore
|
|
rule = Rule(
|
|
topic_id=mapped_topic,
|
|
project_id=mapped_proj,
|
|
title=r_data.get("title", ""),
|
|
statement=r_data.get("statement", ""),
|
|
why=r_data.get("why") or None,
|
|
how_to_apply=r_data.get("how_to_apply") or None,
|
|
order_index=r_data.get("order_index", 0),
|
|
created_at=_dt(r_data.get("created_at")),
|
|
updated_at=_dt(r_data.get("updated_at")),
|
|
)
|
|
session.add(rule)
|
|
await session.flush()
|
|
rule_id_map[r_data["id"]] = rule.id
|
|
stats["rules"] += 1
|
|
|
|
# 12. Rulebook subscriptions (v3 join table)
|
|
for sub in data.get("rulebook_subscriptions", []):
|
|
mapped_pid = project_id_map.get(sub.get("project_id", 0))
|
|
mapped_rbid = rulebook_id_map.get(sub.get("rulebook_id", 0))
|
|
if mapped_pid is None or mapped_rbid is None:
|
|
continue
|
|
await session.execute(project_rulebook_subscriptions.insert().values(
|
|
project_id=mapped_pid, rulebook_id=mapped_rbid,
|
|
))
|
|
stats["rulebook_subscriptions"] += 1
|
|
|
|
# 13. Rule suppressions (v3 join table)
|
|
for sup in data.get("rule_suppressions", []):
|
|
mapped_pid = project_id_map.get(sup.get("project_id", 0))
|
|
mapped_rid = rule_id_map.get(sup.get("rule_id", 0))
|
|
if mapped_pid is None or mapped_rid is None:
|
|
continue
|
|
await session.execute(project_rule_suppressions.insert().values(
|
|
project_id=mapped_pid, rule_id=mapped_rid,
|
|
))
|
|
stats["rule_suppressions"] += 1
|
|
|
|
# 14. Topic suppressions (v3 join table)
|
|
for sup in data.get("topic_suppressions", []):
|
|
mapped_pid = project_id_map.get(sup.get("project_id", 0))
|
|
mapped_tid = topic_id_map.get(sup.get("topic_id", 0))
|
|
if mapped_pid is None or mapped_tid is None:
|
|
continue
|
|
await session.execute(project_topic_suppressions.insert().values(
|
|
project_id=mapped_pid, topic_id=mapped_tid,
|
|
))
|
|
stats["topic_suppressions"] += 1
|
|
|
|
# --- v5 sections. Every one is data.get()-guarded, so a v2/v3/v4
|
|
# payload restores without them rather than failing on an absent key.
|
|
|
|
# 15. Systems
|
|
system_id_map: dict[int, int] = {}
|
|
for sy_data in data.get("systems", []):
|
|
mapped_uid = user_id_map.get(sy_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(sy_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
system = System(
|
|
user_id=mapped_uid, project_id=mapped_pid,
|
|
name=sy_data.get("name", ""),
|
|
description=sy_data.get("description"),
|
|
color=sy_data.get("color"),
|
|
status=sy_data.get("status", "active"),
|
|
order_index=sy_data.get("order_index", 0),
|
|
)
|
|
session.add(system)
|
|
await session.flush()
|
|
system_id_map[sy_data["id"]] = system.id
|
|
stats["systems"] += 1
|
|
|
|
# 16. Record↔system links
|
|
for rs in data.get("record_systems", []):
|
|
mapped_nid = note_id_map.get(rs.get("note_id", 0))
|
|
mapped_sid = system_id_map.get(rs.get("system_id", 0))
|
|
if mapped_nid is None or mapped_sid is None:
|
|
continue
|
|
session.add(RecordSystem(note_id=mapped_nid, system_id=mapped_sid))
|
|
stats["record_systems"] += 1
|
|
|
|
# 16b. Supersession claims. Guarded by `data.get` like every other
|
|
# post-v2 section, so a v5 or older payload restores cleanly without it.
|
|
#
|
|
# Both ends must map. A claim is about a PAIR — half of one is not a
|
|
# weaker claim, it is a dangling row pointing at whatever note happens
|
|
# to hold that id next.
|
|
for sup in data.get("note_supersessions", []):
|
|
mapped_new = note_id_map.get(sup.get("superseder_id", 0))
|
|
mapped_old = note_id_map.get(sup.get("superseded_id", 0))
|
|
if mapped_new is None or mapped_old is None or mapped_new == mapped_old:
|
|
continue
|
|
session.add(
|
|
NoteSupersession(
|
|
superseder_id=mapped_new, superseded_id=mapped_old
|
|
)
|
|
)
|
|
stats["note_supersessions"] += 1
|
|
|
|
# 17. Design systems. The export orders these parent-first, so a
|
|
# parent's new id is always in the map by the time a child needs it —
|
|
# no second pass, and a child whose parent is missing lands as a root
|
|
# rather than failing the whole restore.
|
|
design_system_id_map: dict[int, int] = {}
|
|
for ds_data in data.get("design_systems", []):
|
|
mapped_uid = user_id_map.get(ds_data.get("owner_user_id", 0))
|
|
if mapped_uid is None:
|
|
continue
|
|
design = DesignSystem(
|
|
owner_user_id=mapped_uid,
|
|
title=ds_data.get("title", ""),
|
|
description=ds_data.get("description"),
|
|
guidance=ds_data.get("guidance"),
|
|
parent_id=design_system_id_map.get(ds_data.get("parent_id") or 0),
|
|
)
|
|
session.add(design)
|
|
await session.flush()
|
|
design_system_id_map[ds_data["id"]] = design.id
|
|
stats["design_systems"] += 1
|
|
|
|
# 18. Design tokens
|
|
for t_data in data.get("design_tokens", []):
|
|
mapped_dsid = design_system_id_map.get(t_data.get("design_system_id", 0))
|
|
if mapped_dsid is None:
|
|
continue
|
|
session.add(DesignToken(
|
|
design_system_id=mapped_dsid,
|
|
name=t_data.get("name", ""),
|
|
value_by_mode=t_data.get("value_by_mode") or {},
|
|
group_name=t_data.get("group_name"),
|
|
purpose=t_data.get("purpose"),
|
|
rationale=t_data.get("rationale"),
|
|
supersedes=t_data.get("supersedes") or [],
|
|
order_index=t_data.get("order_index", 0),
|
|
))
|
|
stats["design_tokens"] += 1
|
|
|
|
# 19. Usage events. Kept because pull-through is the evidence base for
|
|
# whether recall works at all, and it is only ever accumulated — a
|
|
# restore that dropped it would silently reset that measurement to zero
|
|
# while everything still looked fine.
|
|
for ev in data.get("note_usage_events", []):
|
|
mapped_nid = note_id_map.get(ev.get("note_id", 0))
|
|
if mapped_nid is None:
|
|
continue
|
|
session.add(NoteUsageEvent(
|
|
user_id=user_id_map.get(ev.get("user_id") or 0),
|
|
note_id=mapped_nid,
|
|
event=ev.get("event", ""),
|
|
source=ev.get("source", ""),
|
|
created_at=_dt(ev.get("created_at")),
|
|
))
|
|
stats["note_usage_events"] += 1
|
|
|
|
# 20. Repo bindings — small, but losing them means every bound repo
|
|
# quietly stops loading its project at session start.
|
|
for rb_data in data.get("repo_bindings", []):
|
|
mapped_uid = user_id_map.get(rb_data.get("user_id", 0))
|
|
mapped_pid = project_id_map.get(rb_data.get("project_id", 0))
|
|
if mapped_uid is None or mapped_pid is None:
|
|
continue
|
|
session.add(RepoBinding(
|
|
user_id=mapped_uid, project_id=mapped_pid,
|
|
repo_key=rb_data.get("repo_key", ""),
|
|
))
|
|
stats["repo_bindings"] += 1
|
|
|
|
await session.commit()
|
|
|
|
logger.info("Restored v2/v3 backup: %s", stats)
|
|
return stats
|