Backups: a daily dedupe-friendly tar to a folder or S3, with restore (#5918-#5921)

#5918 "Backup file": an uncompressed tar of per-table JSON Lines (primary-key
order, sorted keys) plus every attachment file, with fixed tar headers, a
dated name, and an atomic write through a dot-named .part file. Unchanged
rows are the same bytes every day, so restic off-site stores only the edits.
No credential ever enters a backup.

#5919 "Backup destinations": a folder, or an S3-compatible bucket through a
stdlib SigV4 put/list client (checked against AWS's published signatures),
with an upload limit. Test destination only lists, so an append-only key works.

#5920 "Backup job": daily at a chosen hour, Back up now, a 1/2/4 h retry
ladder, a one-hour deadline, an advisory lock, the last 30 runs, and an
overdue verdict on Settings -> Backups.

#5921 "Restore": python -m inkwell.restore FILE into an empty instance, an
integration test that backs up and restores, and docs/backups.md with
Fabled Archives as the worked example.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_backup.py

@@ -0,0 +1,172 @@
+import io
+import tarfile
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from inkwell import backup_file
+from inkwell.backup_data import NOT_INCLUDED, SETTINGS_NOT_INCLUDED, exported_tables
+from inkwell.backup_file import ArchiveError, Contents, read_archive, read_rows, safe_media_path, write_archive
+from inkwell.backup_job import TRIES_PER_DAY, is_due, last_slot
+from inkwell.models import Base
+from inkwell.settings import REGISTRY, SECRET_KEY_SETTING
+
+AT = datetime(2026, 10, 16, 2, 0, tzinfo=timezone.utc)
+
+
+def _contents(tmp_path, notes=None):
+    media = tmp_path / "media"
+    (media / "ab").mkdir(parents=True, exist_ok=True)
+    (media / "ab" / "photo.png").write_bytes(b"\x89PNG not really")
+    return Contents(
+        tables={
+            "notes": notes or [{"id": "1", "body": "milk"}, {"id": "2", "body": "eggs"}],
+            "labels": [{"id": "9", "name": "grocery"}],
+        },
+        files={"ab/photo.png": media / "ab" / "photo.png"},
+    )
+
+
+def _members(path) -> dict[str, bytes]:
+    with tarfile.open(path) as tar:
+        return {m.name: tar.extractfile(m).read() for m in tar.getmembers()}
+
+
+# ---- what goes in ---------------------------------------------------------------------
+
+
+def test_every_table_is_backed_up_or_named_as_left_out():
+    """A table added later must not silently go missing from backups."""
+    every = {t.name for t in Base.metadata.sorted_tables}
+    backed_up = {t.name for t in exported_tables()}
+    assert set(NOT_INCLUDED) <= every, "NOT_INCLUDED names a table that doesn't exist"
+    assert backed_up | set(NOT_INCLUDED) == every
+
+
+def test_no_credential_ever_enters_a_backup():
+    secrets = {d.key for d in REGISTRY if d.secret}
+    assert "backup_s3_secret_key" in secrets and "smtp_password" in secrets
+    assert secrets | {SECRET_KEY_SETTING} <= SETTINGS_NOT_INCLUDED
+
+
+# ---- the file ---------------------------------------------------------------------------
+
+
+def test_unchanged_data_writes_the_same_bytes_on_another_day(tmp_path):
+    out1, out2 = tmp_path / "one", tmp_path / "two"
+    out1.mkdir()
+    out2.mkdir()
+    first, _ = write_archive(_contents(tmp_path), out1, AT, app_version="a")
+    second, _ = write_archive(_contents(tmp_path), out2, AT + timedelta(days=1), app_version="b")
+    a, b = _members(first), _members(second)
+    assert a.keys() == b.keys()
+    for name in a:
+        if name != "manifest.json":
+            assert a[name] == b[name], name
+    assert a["manifest.json"] != b["manifest.json"]
+
+
+def test_one_edit_changes_one_line_of_one_member(tmp_path):
+    (tmp_path / "one").mkdir()
+    (tmp_path / "two").mkdir()
+    before, _ = write_archive(_contents(tmp_path), tmp_path / "one", AT, app_version="a")
+    edited = [{"id": "1", "body": "oat milk"}, {"id": "2", "body": "eggs"}]
+    after, _ = write_archive(_contents(tmp_path, edited), tmp_path / "two", AT, app_version="a")
+    a, b = _members(before), _members(after)
+    changed = [n for n in a if a[n] != b[n]]
+    assert changed == ["tables/notes.jsonl"]
+    lines_a, lines_b = a[changed[0]].splitlines(), b[changed[0]].splitlines()
+    assert sum(x != y for x, y in zip(lines_a, lines_b)) == 1
+
+
+def test_members_come_in_a_fixed_order_with_fixed_headers(tmp_path):
+    (tmp_path / "out").mkdir()
+    path, _ = write_archive(_contents(tmp_path), tmp_path / "out", AT, app_version="a")
+    with tarfile.open(path) as tar:
+        members = tar.getmembers()
+    assert [m.name for m in members] == [
+        "manifest.json",
+        "tables/labels.jsonl",
+        "tables/notes.jsonl",
+        "files/ab/photo.png",
+    ]
+    assert {(m.mtime, m.uid, m.gid, m.mode) for m in members} == {(0, 0, 0, 0o644)}
+    assert path.name == "inkwell-20261016T020000Z.tar"
+
+
+def test_nothing_appears_under_the_final_name_when_writing_fails(tmp_path):
+    contents = _contents(tmp_path)
+    contents.files["gone.png"] = tmp_path / "media" / "gone.png"  # stat() fails
+    (tmp_path / "out").mkdir()
+    with pytest.raises(FileNotFoundError):
+        write_archive(contents, tmp_path / "out", AT, app_version="a")
+    assert list((tmp_path / "out").iterdir()) == []
+
+
+def test_a_backup_reads_back_as_it_was_written(tmp_path):
+    (tmp_path / "out").mkdir()
+    path, _ = write_archive(_contents(tmp_path), tmp_path / "out", AT, app_version="v1")
+    with open(path, "rb") as f:
+        manifest, members = read_archive(f)
+        got = {}
+        for m in members:
+            got[(m.kind, m.name)] = read_rows(m) if m.kind == "table" else m.fileobj.read()
+    assert manifest["app_version"] == "v1"
+    assert manifest["tables"] == {"labels": 1, "notes": 2}
+    assert got[("table", "notes")][0] == {"body": "milk", "id": "1"}
+    assert got[("file", "ab/photo.png")] == b"\x89PNG not really"
+
+
+def test_a_file_that_is_not_a_backup_is_refused():
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w") as tar:
+        tar.addfile(tarfile.TarInfo("other.txt"), io.BytesIO(b""))
+    buf.seek(0)
+    with pytest.raises(ArchiveError):
+        read_archive(buf)
+
+
+@pytest.mark.parametrize("name", ["../etc/passwd", "/etc/passwd", "a/../../b", ""])
+def test_a_file_path_that_leaves_the_media_folder_is_refused(name):
+    with pytest.raises(ArchiveError):
+        safe_media_path(name)
+
+
+# ---- the schedule ---------------------------------------------------------------------
+
+
+def _run(at: datetime, ok: bool) -> dict:
+    return {"started_at": at.isoformat(), "finished_at": (at + timedelta(minutes=1)).isoformat(), "ok": ok}
+
+
+def test_the_slot_is_the_most_recent_scheduled_hour():
+    assert last_slot(datetime(2026, 10, 16, 1, 59, tzinfo=timezone.utc), 2) == datetime(
+        2026, 10, 15, 2, 0, tzinfo=timezone.utc
+    )
+    assert last_slot(AT, 2) == AT
+
+
+def test_due_once_per_slot_and_a_failure_is_tried_again_later_not_at_once():
+    now = AT + timedelta(minutes=5)
+    assert is_due(now, 2, []) is True
+    assert is_due(now, 2, [_run(AT, ok=True)]) is False
+    # Yesterday's success doesn't count for today.
+    assert is_due(now, 2, [_run(AT - timedelta(days=1), ok=True)]) is True
+
+    failed = [_run(AT, ok=False)]
+    assert is_due(AT + timedelta(minutes=30), 2, failed) is False
+    assert is_due(AT + timedelta(hours=1, minutes=2), 2, failed) is True
+
+    # The ladder doubles: 1 h, then 2 h after the second failure.
+    twice = [_run(AT + timedelta(hours=1), ok=False), _run(AT, ok=False)]
+    assert is_due(AT + timedelta(hours=2), 2, twice) is False
+    assert is_due(AT + timedelta(hours=3, minutes=2), 2, twice) is True
+
+    # And stops for the day after its tries are used.
+    spent = [_run(AT + timedelta(hours=h), ok=False) for h in range(TRIES_PER_DAY)][::-1]
+    assert is_due(AT + timedelta(hours=20), 2, spent) is False
+
+
+def test_the_archive_name_prefix_matches_what_a_destination_counts():
+    assert backup_file.archive_name(AT).startswith(backup_file.NAME_PREFIX)
+    assert backup_file.archive_name(AT).endswith(backup_file.NAME_SUFFIX)

tests/test_backup_s3.py

@@ -0,0 +1,110 @@
+import threading
+from datetime import datetime, timezone
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from inkwell.backup_s3 import EMPTY_SHA256, S3Error, S3Target, list_objects, put_file, sign
+
+# AWS's published Signature Version 4 examples for S3 ("Signature Calculations for the
+# Authorization Header: Transferring Payload in a Single Chunk").
+AK = "AKIAIOSFODNN7EXAMPLE"
+SK = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+AT = datetime(2013, 5, 24, tzinfo=timezone.utc)
+
+
+def _signature(headers: dict) -> str:
+    return headers["Authorization"].rsplit("Signature=", 1)[1]
+
+
+def test_get_object_matches_aws_example():
+    headers = sign(
+        "GET", "https://examplebucket.s3.amazonaws.com/test.txt", {"Range": "bytes=0-9"},
+        EMPTY_SHA256, AK, SK, "us-east-1", AT,
+    )
+    assert "SignedHeaders=host;range;x-amz-content-sha256;x-amz-date" in headers["Authorization"]
+    assert _signature(headers) == "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+
+
+def test_list_objects_matches_aws_example():
+    headers = sign(
+        "GET", "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", {},
+        EMPTY_SHA256, AK, SK, "us-east-1", AT,
+    )
+    assert _signature(headers) == "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+
+
+# ---- against a local stand-in bucket ---------------------------------------------------
+
+LISTING = b"""<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Contents><Key>inkwell-20261015T020000Z.tar</Key><Size>42</Size><LastModified>2026-10-15T02:00:05Z</LastModified></Contents>
+</ListBucketResult>"""
+
+DENIED = b"""<?xml version="1.0" encoding="UTF-8"?>
+<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"""
+
+
+@pytest.fixture
+def bucket():
+    received: dict = {"puts": {}, "deny": False}
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_PUT(self):
+            length = int(self.headers["Content-Length"])
+            received["puts"][self.path] = (self.rfile.read(length), dict(self.headers))
+            if received["deny"]:
+                self.send_response(403)
+                self.end_headers()
+                self.wfile.write(DENIED)
+                return
+            self.send_response(200)
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+
+        def do_GET(self):
+            received["get"] = self.path
+            self.send_response(200)
+            self.send_header("Content-Length", str(len(LISTING)))
+            self.end_headers()
+            self.wfile.write(LISTING)
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    received["target"] = S3Target(f"http://127.0.0.1:{server.server_address[1]}", "backups", AK, SK)
+    yield received
+    server.shutdown()
+
+
+def test_a_file_is_put_whole_and_signed(bucket, tmp_path):
+    path = tmp_path / "inkwell-x.tar"
+    path.write_bytes(b"x" * 300_000)
+    put_file(bucket["target"], "inkwell-x.tar", path, limit_kbps=100_000)
+    body, headers = bucket["puts"]["/backups/inkwell-x.tar"]
+    assert body == b"x" * 300_000
+    assert headers["Authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={AK}/")
+    assert headers["Host"] == bucket["target"].endpoint.split("//", 1)[1]
+
+
+def test_a_refusal_says_why_in_words(bucket, tmp_path):
+    bucket["deny"] = True
+    path = tmp_path / "inkwell-x.tar"
+    path.write_bytes(b"x")
+    with pytest.raises(S3Error, match="AccessDenied: the key isn't allowed to do that"):
+        put_file(bucket["target"], "inkwell-x.tar", path)
+
+
+def test_listing_reads_the_objects(bucket):
+    objects = list_objects(bucket["target"], "inkwell-")
+    assert [(o.key, o.size) for o in objects] == [("inkwell-20261015T020000Z.tar", 42)]
+    assert "list-type=2" in bucket["get"] and "prefix=inkwell-" in bucket["get"]
+
+
+def test_an_address_nothing_answers_on_says_so():
+    target = S3Target("http://127.0.0.1:1", "backups", AK, SK)
+    with pytest.raises(S3Error, match="Couldn't reach"):
+        list_objects(target)

tests/test_integration.py

@@ -2099,3 +2099,99 @@ async def test_the_test_email_goes_to_the_admin_and_reports_a_failure(app_client
     failed = await app_client.post("/api/settings/test-email")
     assert failed.status_code == 502
     assert "bad credentials" in (await failed.get_json())["error"]
+
+
+# --- backups (#5918–#5921) --------------------------------------------------------------
+
+
+async def _backup_settings_off() -> None:
+    """Settings outlive the per-test truncate, so each backup test starts from none."""
+    async with session_scope() as fresh:
+        await fresh.execute(delete(Setting).where(Setting.key.like("backup%")))
+        await fresh.execute(delete(Setting).where(Setting.key == "smtp_password"))
+        await fresh.commit()
+
+
+async def test_a_backup_is_written_to_a_folder_and_restores_into_an_empty_instance(app_client, db, tmp_path):
+    import tarfile
+
+    from inkwell import backup_job
+    from inkwell.restore import restore
+
+    await _backup_settings_off()
+    await _admin_with_invite(app_client)
+    note = await (await app_client.post("/api/notes", json={"body": "buy milk #grocery"})).get_json()
+    uploaded = await app_client.post(f"/api/notes/{note['id']}/attachments", files=_upload(b"photo bytes", "p.bin"))
+    assert uploaded.status_code == 201
+
+    folder = tmp_path / "backups"
+    folder.mkdir()
+    saved = await app_client.patch(
+        "/api/settings",
+        json={"backup_destination": "folder", "backup_folder": str(folder), "smtp_password": "hunter2"},
+    )
+    assert saved.status_code == 200, await saved.get_data(as_text=True)
+
+    checked = await app_client.post("/api/backups/test")
+    assert checked.status_code == 200, await checked.get_data(as_text=True)
+    assert "writable and holds 0 Inkwell backups" in (await checked.get_json())["message"]
+
+    assert (await app_client.post("/api/backups/run")).status_code == 202
+    await backup_job.drain()
+    status = await (await app_client.get("/api/backups")).get_json()
+    run = status["runs"][0]
+    assert run["ok"] is True, run
+    assert run["files"] == 1 and run["missing_files"] == 0
+    path = folder / run["name"]
+    assert path.is_file() and not list(folder.glob(".*"))
+
+    # No credential travels in a backup.
+    with tarfile.open(path) as tar:
+        settings_rows = tar.extractfile("tables/settings.jsonl").read().decode()
+        names = tar.getnames()
+    assert "hunter2" not in settings_rows and '"secret_key"' not in settings_rows
+    assert '"backup_folder"' in settings_rows
+    assert "tables/password_resets.jsonl" not in names
+
+    # A different instance: the database emptied and the attachments gone.
+    async with session_scope() as fresh:
+        await fresh.execute(text(f"TRUNCATE {_TABLES}, device_tokens RESTART IDENTITY CASCADE"))
+        await fresh.commit()
+    for f in Config.media_root().rglob("*"):
+        if f.is_file():
+            f.unlink()
+
+    counts = await restore(path)
+    assert counts["files"] == 1 and counts["notes"] == 1
+
+    async with session_scope() as fresh:
+        back = await fresh.get(Note, uuid.UUID(note["id"]))
+        assert back.body == "buy milk #grocery"
+        tags = (await fresh.execute(select(NoteLabel).where(NoteLabel.note_id == back.id))).scalars().all()
+        assert len(tags) == 1
+        att = await fresh.scalar(select(NoteAttachment).where(NoteAttachment.note_id == back.id))
+        assert (Config.media_root() / att.path).read_bytes() == b"photo bytes"
+
+    # And it refuses an instance that already has accounts.
+    with pytest.raises(SystemExit):
+        await restore(path)
+    await _backup_settings_off()
+
+
+async def test_back_up_now_says_what_is_wrong_with_the_destination(app_client, db, tmp_path):
+    from inkwell import backup_job
+
+    await _backup_settings_off()
+    await _admin_with_invite(app_client)
+    await app_client.patch("/api/settings", json={"backup_folder": str(tmp_path / "not-mounted")})
+
+    tested = await app_client.post("/api/backups/test")
+    assert tested.status_code == 400
+    assert "isn't there" in (await tested.get_json())["error"]
+
+    assert (await app_client.post("/api/backups/run")).status_code == 202
+    await backup_job.drain()
+    status = await (await app_client.get("/api/backups")).get_json()
+    assert status["runs"][0]["ok"] is False
+    assert "isn't there" in status["runs"][0]["error"]
+    await _backup_settings_off()