feat(523): the backup as an uncompressed tar of per-table JSON Lines, ordered so restic dedupes it

export_full_backup orders every query by primary key: an UPDATE moves a
row to the end of the heap, so an unordered export shifted every byte
after an edited row. services/backup_file.py writes the payload as one
dated tar per run - manifest.json plus <section>.jsonl, sorted keys,
fixed member headers - atomically via a dot-prefixed .part file, and
reads it back into the dict restore_full_backup takes, refusing a
missing or short section.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_integration_backup_file_ordering.py

@@ -0,0 +1,94 @@
+"""Real-Postgres check that the full export is ORDERED, so the backup file
+deduplicates between runs (milestone 523).
+
+Only Postgres shows this. An unordered select returns rows in heap order, and
+an UPDATE writes the row's new version at the end of the heap — so before
+export_full_backup ordered its queries, editing one old note moved it to the
+bottom of `notes`. Every byte after its old place shifted, and a restic run
+over the next backup stored most of the table again. A mocked session returns
+whatever order it was handed and could never catch it.
+"""
+import io
+import tarfile
+
+import pytest
+import pytest_asyncio
+from sqlalchemy import select
+
+from scribe.models import async_session
+from scribe.models.note import Note
+from scribe.models.user import User
+from scribe.services import backup
+from scribe.services.backup_file import write_archive_to
+from tests.helpers import ensure_user
+
+pytestmark = [pytest.mark.integration, pytest.mark.usefixtures("_dispose_engine")]
+
+USERNAME = "backup_file_ordering_owner"
+
+
+async def _purge() -> None:
+    async with async_session() as s:
+        for user in (await s.execute(
+            select(User).where(User.username == USERNAME)
+        )).scalars().all():
+            for note in (await s.execute(
+                select(Note).where(Note.user_id == user.id)
+            )).scalars().all():
+                await s.delete(note)
+            await s.delete(user)
+        await s.commit()
+
+
+@pytest_asyncio.fixture(autouse=True)
+async def _no_leftovers():
+    """Setup only — see the rule_version roundtrip module for why a teardown
+    here would break the next test's event loop."""
+    await _purge()
+
+
+@pytest_asyncio.fixture
+async def notes():
+    async with async_session() as s:
+        user = await ensure_user(s, USERNAME)
+        rows = [Note(user_id=user.id, title=f"ordering {n}", body="b" * 100)
+                for n in range(5)]
+        s.add_all(rows)
+        await s.commit()
+        ids = [n.id for n in rows]
+    yield ids
+    await _purge()
+
+
+def _sections(payload: dict) -> dict[str, bytes]:
+    buf = io.BytesIO()
+    write_archive_to(payload, buf, app_version="test")
+    buf.seek(0)
+    with tarfile.open(fileobj=buf, mode="r:") as tar:
+        return {m.name: tar.extractfile(m).read() for m in tar.getmembers()
+                if m.name != "manifest.json"}
+
+
+async def _edit_first(ids: list[int]) -> None:
+    async with async_session() as s:
+        note = await s.get(Note, ids[0])
+        note.body = "edited after the first export"
+        await s.commit()
+
+
+async def test_an_edited_row_keeps_its_place(notes):
+    await _edit_first(notes)
+    payload = await backup.export_full_backup()
+    exported = [row["id"] for row in payload["notes"]]
+    assert exported == sorted(exported)
+    mine = [i for i in exported if i in set(notes)]
+    assert mine == notes
+
+
+async def test_two_exports_of_unchanged_data_are_the_same_bytes(notes):
+    await _edit_first(notes)
+    first = _sections(await backup.export_full_backup())
+    second = _sections(await backup.export_full_backup())
+    assert first.keys() == second.keys()
+    for name in first:
+        assert first[name] == second[name], name

tests/test_services_backup_file.py

@@ -0,0 +1,201 @@
+"""Unit tests for the backup file (services/backup_file.py).
+
+The file exists to be deduplicated between runs by a tool like restic, so
+most of what is tested here is a property of BYTES: what stays identical
+when nothing changed, and how little moves when one row does. A round trip
+alone would pass for a gzip of one big JSON document, which is exactly the
+shape this file replaced.
+"""
+import io
+import tarfile
+from datetime import datetime, timezone
+from unittest.mock import patch
+
+import pytest
+
+from scribe.services import backup_file
+from scribe.services.backup_file import (
+    ArchiveError, archive_name, is_archive, read_archive, write_archive,
+    write_archive_to,
+)
+
+AT = datetime(2026, 10, 15, 3, 0, 0, tzinfo=timezone.utc)
+
+
+def _payload(**over) -> dict:
+    payload = {
+        "version": 25,
+        "scope": "full",
+        "exported_at": "2026-10-15T03:00:00+00:00",
+        "_security_notice": "hashed passwords inside",
+        "_not_included": ["api_keys", "note_embeddings"],
+        "users": [{"id": 1, "username": "ada", "role": "admin"}],
+        "notes": [
+            {"id": n, "user_id": 1, "title": f"note {n}", "body": "x" * 200,
+             "tags": ["a", "b"]}
+            for n in range(1, 51)
+        ],
+        "settings": [],
+    }
+    payload.update(over)
+    return payload
+
+
+def _members(payload: dict) -> dict[str, bytes]:
+    buf = io.BytesIO()
+    write_archive_to(payload, buf, app_version="2026.10.15.0300")
+    buf.seek(0)
+    with tarfile.open(fileobj=buf, mode="r:") as tar:
+        return {m.name: tar.extractfile(m).read() for m in tar.getmembers()}
+
+
+def test_the_name_is_dated_in_utc_and_sorts():
+    assert archive_name(AT) == "scribe-20261015T030000Z.tar"
+    local = datetime(2026, 10, 14, 23, 0, 0).astimezone()
+    assert archive_name(local).endswith("Z.tar")
+
+
+def test_round_trip_returns_the_payload_restore_takes():
+    payload = _payload()
+    buf = io.BytesIO()
+    write_archive_to(payload, buf, app_version="v")
+    buf.seek(0)
+    assert read_archive(buf) == payload
+
+
+def test_notices_stay_in_the_manifest_not_as_sections():
+    """`_not_included` is a list, but of table names; it must not become a
+    member that a restore would read as rows."""
+    names = set(_members(_payload()))
+    assert names == {"manifest.json", "notes.jsonl", "settings.jsonl", "users.jsonl"}
+
+
+def test_members_come_in_one_order_with_fixed_headers():
+    buf = io.BytesIO()
+    write_archive_to(_payload(), buf, app_version="v")
+    buf.seek(0)
+    with tarfile.open(fileobj=buf, mode="r:") as tar:
+        members = tar.getmembers()
+    assert [m.name for m in members] == [
+        "manifest.json", "notes.jsonl", "settings.jsonl", "users.jsonl",
+    ]
+    for m in members:
+        assert (m.mtime, m.uid, m.gid, m.uname, m.gname, m.mode) == (
+            0, 0, 0, "", "", 0o644,
+        )
+
+
+def test_an_unchanged_payload_gives_identical_sections_on_another_run():
+    """The run's own time and the app version live in the manifest and
+    nowhere else, so a second run over unchanged data repeats every section
+    byte for byte."""
+    first = _members(_payload())
+    second = _members(_payload(exported_at="2026-10-16T03:00:00+00:00"))
+    assert first["manifest.json"] != second["manifest.json"]
+    for name in first:
+        if name != "manifest.json":
+            assert first[name] == second[name], name
+
+
+def test_key_order_in_a_row_does_not_change_its_bytes():
+    rows = _payload()["notes"]
+    shuffled = [dict(reversed(list(r.items()))) for r in rows]
+    assert _members(_payload(notes=shuffled))["notes.jsonl"] == \
+        _members(_payload())["notes.jsonl"]
+
+
+def test_editing_one_row_changes_one_line_of_one_member():
+    before = _members(_payload())
+    notes = _payload()["notes"]
+    notes[24] = {**notes[24], "body": "edited"}
+    after = _members(_payload(notes=notes))
+
+    changed = [n for n in before if before[n] != after[n]]
+    assert changed == ["notes.jsonl"]
+    old_lines = before["notes.jsonl"].splitlines()
+    new_lines = after["notes.jsonl"].splitlines()
+    assert len(old_lines) == len(new_lines)
+    assert [i for i, (a, b) in enumerate(zip(old_lines, new_lines)) if a != b] == [24]
+
+
+def test_a_new_row_appends_to_the_end_of_its_member():
+    """Rows come in primary-key order, so a table that grows grows at its
+    tail: everything already sent is a prefix of the new member."""
+    before = _members(_payload())["notes.jsonl"]
+    notes = _payload()["notes"] + [{"id": 51, "user_id": 1, "title": "new"}]
+    after = _members(_payload(notes=notes))["notes.jsonl"]
+    assert after.startswith(before)
+    assert after.count(b"\n") == before.count(b"\n") + 1
+
+
+def test_the_file_is_an_uncompressed_tar():
+    buf = io.BytesIO()
+    write_archive_to(_payload(), buf, app_version="v")
+    raw = buf.getvalue()
+    assert raw[257:262] == b"ustar"
+    assert b'"title":"note 7"' in raw
+
+
+def test_is_archive_tells_a_tar_from_the_json_download():
+    buf = io.BytesIO()
+    write_archive_to(_payload(), buf, app_version="v")
+    buf.seek(0)
+    assert is_archive(buf)
+    assert buf.tell() == 0
+    assert not is_archive(io.BytesIO(b'{"version": 25, "users": []}'))
+
+
+def test_write_archive_lands_under_the_dated_name_only(tmp_path):
+    path = write_archive(_payload(), tmp_path, AT, app_version="v")
+    assert path == tmp_path / "scribe-20261015T030000Z.tar"
+    assert [p.name for p in tmp_path.iterdir()] == [path.name]
+    with open(path, "rb") as f:
+        assert read_archive(f) == _payload()
+
+
+def test_a_failed_write_leaves_nothing_behind(tmp_path):
+    """Neither a half backup under the final name (a watcher would count it
+    as arrived) nor a stray part file."""
+    with patch.object(backup_file, "write_archive_to", side_effect=OSError("disk full")):
+        with pytest.raises(OSError):
+            write_archive(_payload(), tmp_path, AT, app_version="v")
+    assert list(tmp_path.iterdir()) == []
+
+
+def _rewrite(members: dict[str, bytes]) -> io.BytesIO:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w", format=tarfile.USTAR_FORMAT) as tar:
+        for name, data in members.items():
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tar.addfile(info, io.BytesIO(data))
+    buf.seek(0)
+    return buf
+
+
+def test_a_missing_section_is_an_error_not_a_thinner_restore():
+    members = _members(_payload())
+    del members["notes.jsonl"]
+    with pytest.raises(ArchiveError, match="notes.jsonl"):
+        read_archive(_rewrite(members))
+
+
+def test_a_section_cut_short_is_an_error():
+    members = _members(_payload())
+    members["notes.jsonl"] = b"\n".join(members["notes.jsonl"].splitlines()[:10]) + b"\n"
+    with pytest.raises(ArchiveError, match="10 rows"):
+        read_archive(_rewrite(members))
+
+
+def test_a_newer_format_is_refused_by_name():
+    members = _members(_payload())
+    members["manifest.json"] = members["manifest.json"].replace(
+        b'"format_version": 1', b'"format_version": 99',
+    )
+    with pytest.raises(ArchiveError, match="upgrade"):
+        read_archive(_rewrite(members))
+
+
+def test_garbage_is_not_an_archive():
+    with pytest.raises(ArchiveError):
+        read_archive(io.BytesIO(b"not a tar at all" * 100))