bvandeusen/FabledGrimoire
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

feat(523): the backup as an uncompressed tar of per-table JSON Lines, ordered so restic dedupes it
CI & Build / Python lint (push) Successful in 4s
Details
CI & Build / Plugin hooks (push) Successful in 13s
Details
CI & Build / TypeScript typecheck (push) Successful in 55s
Details
CI & Build / integration (push) Successful in 1m9s
Details
CI & Build / Python tests (push) Successful in 1m55s
Details
CI & Build / Build & push image (push) Successful in 1m36s
Details

Browse Source 
export_full_backup orders every query by primary key: an UPDATE moves a
row to the end of the heap, so an unordered export shifted every byte
after an edited row. services/backup_file.py writes the payload as one
dated tar per run - manifest.json plus <section>.jsonl, sorted keys,
fixed member headers - atomically via a dot-prefixed .part file, and
reads it back into the dict restore_full_backup takes, refusing a
missing or short section.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-15 16:30:16 -04:00
co-authored by Claude Opus 5.5
parent 69dddb7551
commit 1942a7a336
4 changed files with 555 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scribe/services/backup.py
+46 -32
View File
@@ -982,42 +982,53 @@ def _rule_rows(rows) -> list[dict]:
# ---------------------------------------------------------------------------
async def export_full_backup() -> dict:
"""Export all data as a version-5 JSON backup."""
"""Export all data as a full backup payload.
Every query is ORDERED, by primary key unless a restore needs another
order. Postgres returns an unordered select in heap order, and an UPDATE
moves a row to the end of the heap: an edited note would jump to the
bottom of `notes`, shifting every byte after its old place. The scheduled
backup file (services/backup_file.py) is deduplicated by restic between
runs, which only works when unchanged rows serialise to the same bytes in
the same place.
"""
async with async_session() as session:
users = (await session.execute(select(User))).scalars().all()
projects = (await session.execute(select(Project))).scalars().all()
milestones = (await session.execute(select(Milestone))).scalars().all()
notes = (await session.execute(select(Note))).scalars().all()
task_logs = (await session.execute(select(TaskLog))).scalars().all()
note_drafts = (await session.execute(select(NoteDraft))).scalars().all()
users =(await session.execute(select(User).order_by(User.id))).scalars().all()
projects = (await session.execute(select(Project).order_by(Project.id))).scalars().all()
milestones = (await session.execute(select(Milestone).order_by(Milestone.id))).scalars().all()
notes = (await session.execute(select(Note).order_by(Note.id))).scalars().all()
task_logs = (await session.execute(select(TaskLog).order_by(TaskLog.id))).scalars().all()
note_drafts = (await session.execute(select(NoteDraft).order_by(NoteDraft.id))).scalars().all()
note_versions = (await session.execute(
select(NoteVersion).order_by(NoteVersion.note_id, NoteVersion.id)
)).scalars().all()
rule_versions = (await session.execute(
select(RuleVersion).order_by(RuleVersion.rule_id, RuleVersion.id)
)).scalars().all()
settings = (await session.execute(select(Setting))).scalars().all()
systems = (await session.execute(select(System))).scalars().all()
settings = (await session.execute(select(Setting).order_by(Setting.user_id, Setting.key))).scalars().all()
systems = (await session.execute(select(System).order_by(System.id))).scalars().all()
canonical_systems = (await session.execute(
select(CanonicalSystem).where(CanonicalSystem.deleted_at.is_(None))
.order_by(CanonicalSystem.order_index)
.order_by(CanonicalSystem.order_index, CanonicalSystem.id)
)).scalars().all()
rule_system_rows = (await session.execute(
select(rule_systems_t.c.rule_id, CanonicalSystem.slug)
.join(CanonicalSystem, CanonicalSystem.id == rule_systems_t.c.canonical_id)
.order_by(rule_systems_t.c.rule_id, CanonicalSystem.slug)
)).all()
rule_moment_rows = (await session.execute(
select(rule_moments_t.c.rule_id, rule_moments_t.c.moment)
.order_by(rule_moments_t.c.rule_id, rule_moments_t.c.moment)
)).all()
rule_moment_judgments = (await session.execute(
select(RuleMomentJudgment)
select(RuleMomentJudgment).order_by(RuleMomentJudgment.id)
)).scalars().all()
rule_relations = (await session.execute(select(RuleRelation))).scalars().all()
lesson_rule_links = (await session.execute(select(LessonRuleLink))).scalars().all()
lesson_no_rule = (await session.execute(select(LessonNoRule))).scalars().all()
record_systems = (await session.execute(select(RecordSystem))).scalars().all()
rule_relations = (await session.execute(select(RuleRelation).order_by(RuleRelation.id))).scalars().all()
lesson_rule_links = (await session.execute(select(LessonRuleLink).order_by(LessonRuleLink.id))).scalars().all()
lesson_no_rule = (await session.execute(select(LessonNoRule).order_by(LessonNoRule.lesson_id))).scalars().all()
record_systems = (await session.execute(select(RecordSystem).order_by(RecordSystem.id))).scalars().all()
supersessions = (
await session.execute(select(NoteSupersession))
await session.execute(select(NoteSupersession).order_by(NoteSupersession.id))
).scalars().all()
# Parent-first, so a restore can resolve parent_id as it goes rather
# than needing a second pass — the self-FK is the only ordering
@@ -1026,13 +1037,13 @@ async def export_full_backup() -> dict:
select(DesignSystem).order_by(DesignSystem.parent_id.nullsfirst(),
DesignSystem.id)
)).scalars().all()
design_tokens = (await session.execute(select(DesignToken))).scalars().all()
usage_events = (await session.execute(select(NoteUsageEvent))).scalars().all()
design_tokens = (await session.execute(select(DesignToken).order_by(DesignToken.id))).scalars().all()
usage_events = (await session.execute(select(NoteUsageEvent).order_by(NoteUsageEvent.id))).scalars().all()
rule_usage_events = (
await session.execute(select(RuleUsageEvent))
await session.execute(select(RuleUsageEvent).order_by(RuleUsageEvent.id))
).scalars().all()
system_usage_events = (
await session.execute(select(SystemUsageEvent))
await session.execute(select(SystemUsageEvent).order_by(SystemUsageEvent.id))
).scalars().all()
# Oldest first, so a restored history reads in the order the dials
# actually moved — the sequence IS the argument when a surface has been
@@ -1043,30 +1054,33 @@ async def export_full_backup() -> dict:
moment_mappings = (await session.execute(
select(MomentMapping).order_by(MomentMapping.id)
)).scalars().all()
repo_bindings = (await session.execute(select(RepoBinding))).scalars().all()
code_shapes = (await session.execute(select(CodeShape))).scalars().all()
repo_bindings = (await session.execute(select(RepoBinding).order_by(RepoBinding.id))).scalars().all()
code_shapes = (await session.execute(select(CodeShape).order_by(CodeShape.id))).scalars().all()
code_shape_events = (await session.execute(
select(CodeShapeEvent).order_by(CodeShapeEvent.at, CodeShapeEvent.id)
)).scalars().all()
code_shape_uses = (await session.execute(
select(CodeShapeUse).order_by(CodeShapeUse.shape_id, CodeShapeUse.snippet_id)
select(CodeShapeUse).order_by(CodeShapeUse.shape_id, CodeShapeUse.snippet_id, CodeShapeUse.id)
)).scalars().all()
rulebooks = (await session.execute(select(Rulebook))).scalars().all()
topics = (await session.execute(select(RulebookTopic))).scalars().all()
rules = (await session.execute(select(Rule))).scalars().all()
rulebooks = (await session.execute(select(Rulebook).order_by(Rulebook.id))).scalars().all()
topics = (await session.execute(select(RulebookTopic).order_by(RulebookTopic.id))).scalars().all()
rules = (await session.execute(select(Rule).order_by(Rule.id))).scalars().all()
platforms = (await session.execute(
select(Platform).where(Platform.deleted_at.is_(None))
.order_by(Platform.order_index)
.order_by(Platform.order_index, Platform.id)
)).scalars().all()
project_platforms = (await session.execute(select(ProjectPlatform))).scalars().all()
family_ideas = (await session.execute(select(FamilyIdea))).scalars().all()
project_platforms = (await session.execute(select(ProjectPlatform).order_by(
ProjectPlatform.project_id, ProjectPlatform.platform_id))).scalars().all()
family_ideas = (await session.execute(select(FamilyIdea).order_by(FamilyIdea.note_id))).scalars().all()
family_idea_platforms = (await session.execute(
select(FamilyIdeaPlatform)
select(FamilyIdeaPlatform).order_by(
FamilyIdeaPlatform.note_id, FamilyIdeaPlatform.platform_id)
)).scalars().all()
family_idea_references = (await session.execute(
select(FamilyIdeaReference)
select(FamilyIdeaReference).order_by(
FamilyIdeaReference.idea_id, FamilyIdeaReference.snippet_id)
)).scalars().all()
family_adoptions = (await session.execute(select(FamilyAdoption))).scalars().all()
family_adoptions = (await session.execute(select(FamilyAdoption).order_by(FamilyAdoption.id))).scalars().all()
# Oldest first: a precedent is always an earlier decision, so a restore
# in this order has every precedent mapped before anything cites it.
family_decisions = (await session.execute(