feat: restore a new install from the off-site bucket, database first, then every file

Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/fake_s3.py

@@ -0,0 +1,110 @@
+"""An in-process S3 bucket for round-trip tests: put, multipart upload,
+get, and ListObjectsV2, kept in memory. It doesn't check signatures; the
+signing has its own tests against AWS's published vectors. Like Fabled
+Archives, it refuses to replace an object that is already there.
+
+    with FakeS3() as s3:
+        target = s3.target()
+        ...
+        s3.objects  # {key: bytes}
+"""
+from __future__ import annotations
+
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import parse_qs, unquote, urlsplit
+from xml.sax.saxutils import escape
+
+from backend.app.services.backup_s3 import S3Target
+
+BUCKET = "curator"
+
+
+class FakeS3:
+    def __init__(self):
+        self.objects: dict[str, bytes] = {}
+        self.uploads: dict[str, dict[int, bytes]] = {}
+        outer = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def _parts(self):
+                u = urlsplit(self.path)
+                path = unquote(u.path).lstrip("/")
+                bucket, _, key = path.partition("/")
+                query = parse_qs(u.query, keep_blank_values=True)   # `?uploads` has no value
+                return bucket, key, {k: v[0] for k, v in query.items()}
+
+            def _reply(self, status, body=b"", headers=None):
+                self.send_response(status)
+                for name, value in (headers or {}).items():
+                    self.send_header(name, value)
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                if self.command != "HEAD":
+                    self.wfile.write(body)
+
+            def _body(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                return self.rfile.read(length) if length else b""
+
+            def do_PUT(self):
+                _, key, q = self._parts()
+                body = self._body()
+                if "uploadId" in q:
+                    outer.uploads[q["uploadId"]][int(q["partNumber"])] = body
+                    return self._reply(200, headers={"ETag": f'"p{q["partNumber"]}"'})
+                if key in outer.objects:
+                    return self._reply(403, b"<Error><Code>AccessDenied</Code></Error>")
+                outer.objects[key] = body
+                return self._reply(200)
+
+            def do_POST(self):
+                _, key, q = self._parts()
+                self._body()
+                if "uploads" in q:
+                    upload_id = f"u{len(outer.uploads) + 1}"
+                    outer.uploads[upload_id] = {}
+                    return self._reply(200, (
+                        "<InitiateMultipartUploadResult>"
+                        f"<UploadId>{upload_id}</UploadId>"
+                        "</InitiateMultipartUploadResult>").encode())
+                parts = outer.uploads.pop(q["uploadId"])
+                outer.objects[key] = b"".join(parts[n] for n in sorted(parts))
+                return self._reply(200, b"<CompleteMultipartUploadResult/>")
+
+            def do_DELETE(self):
+                _, _, q = self._parts()
+                outer.uploads.pop(q.get("uploadId", ""), None)
+                return self._reply(204)
+
+            def do_GET(self):
+                _, key, q = self._parts()
+                if q.get("list-type") == "2":
+                    prefix = q.get("prefix", "")
+                    items = "".join(
+                        f"<Contents><Key>{escape(k)}</Key><Size>{len(v)}</Size>"
+                        "<LastModified>2026-01-01T00:00:00Z</LastModified></Contents>"
+                        for k, v in sorted(outer.objects.items()) if k.startswith(prefix)
+                    )
+                    return self._reply(200, f"<ListBucketResult>{items}</ListBucketResult>".encode())
+                if key not in outer.objects:
+                    return self._reply(404, b"<Error><Code>NoSuchKey</Code></Error>")
+                return self._reply(200, outer.objects[key])
+
+            def log_message(self, *args):
+                pass
+
+        self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, *exc):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def target(self) -> S3Target:
+        host, port = self.httpd.server_address
+        return S3Target(f"http://{host}:{port}", BUCKET, "AK", "SK")

tests/test_api_system_backup.py

@@ -496,3 +496,91 @@ async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _seal
     )
     assert resp.status_code == 400
     assert "upload_limit_mbps" in (await resp.get_json())["detail"]
+
+
+# ── restoring from the off-site bucket ──────────────────────────────────────
+
+
+@pytest.fixture
+def _saved_bucket(monkeypatch):
+    from backend.app.services import backup_destination
+    from backend.app.services.backup_s3 import S3Target
+
+    target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
+    monkeypatch.setattr(backup_destination, "saved_target", lambda settings: target)
+    dispatched = []
+    monkeypatch.setattr("backend.app.tasks.backup.restore_offsite_task.delay",
+                        lambda **kw: dispatched.append(kw))
+    return dispatched
+
+
+_DUMP = "database/fabledcurator-20261107T134258Z.dump"
+
+
+@pytest.mark.asyncio
+async def test_the_bucket_dumps_are_listed_with_their_confirmation(
+    client, _saved_bucket, monkeypatch,
+):
+    from backend.app.services import backup_restore
+
+    monkeypatch.setattr(backup_restore, "list_dumps", lambda target: [
+        backup_restore.Dump(_DUMP, "20261107T134258Z", 3222868924),
+    ])
+    resp = await client.get("/api/system/backup/offsite/dumps")
+    assert resp.status_code == 200
+    assert (await resp.get_json())["dumps"] == [{
+        "key": _DUMP, "ts": "20261107T134258Z", "size_bytes": 3222868924,
+        "confirm": "restore-offsite-20261107T134258Z",
+    }]
+
+
+@pytest.mark.asyncio
+async def test_a_restore_needs_the_dump_typed_back(client, _saved_bucket):
+    resp = await client.post("/api/system/backup/offsite/restore",
+                             json={"key": _DUMP, "confirm": "yes"})
+    assert resp.status_code == 400
+    assert (await resp.get_json())["expected"] == "restore-offsite-20261107T134258Z"
+    assert _saved_bucket == []
+
+    resp = await client.post("/api/system/backup/offsite/restore", json={
+        "key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
+    assert resp.status_code == 202
+    assert _saved_bucket == [{"dump_key": _DUMP}]
+
+
+@pytest.mark.asyncio
+async def test_a_restore_waits_while_a_backup_runs(client, _saved_bucket, db_sync):
+    db_sync.add(BackupRun(kind="db", status="running", triggered_by="nightly",
+                          started_at=datetime.now(UTC), manifest={}))
+    db_sync.commit()
+    resp = await client.post("/api/system/backup/offsite/restore", json={
+        "key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
+    assert resp.status_code == 409
+    assert "db backup is running" in (await resp.get_json())["detail"]
+    resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
+    assert resp.status_code == 409
+    assert _saved_bucket == []
+
+
+@pytest.mark.asyncio
+async def test_fetching_missing_files_leaves_the_database(client, _saved_bucket):
+    resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
+    assert resp.status_code == 202
+    assert _saved_bucket == [{"files_only": True}]
+
+
+@pytest.mark.asyncio
+async def test_restoring_needs_a_destination(client):
+    resp = await client.get("/api/system/backup/offsite/dumps")
+    assert resp.status_code == 400
+    assert (await resp.get_json())["error"] == "no_destination"
+
+
+@pytest.mark.asyncio
+async def test_restore_runs_are_listed_as_their_own_kind(client, db_sync):
+    db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                          started_at=datetime.now(UTC), manifest={"phase": "files"}))
+    db_sync.commit()
+    resp = await client.get("/api/system/backup/runs?kind=restore")
+    runs = (await resp.get_json())["runs"]
+    assert [r["status"] for r in runs] == ["restoring"]

tests/test_backup_restore.py

@@ -0,0 +1,297 @@
+"""Restoring from the off-site bucket (services/backup_restore.py and
+restore_offsite_task): the database from a dated dump, then every library
+file back at the path its row records, checked against its sha256 on arrival.
+The round trip at the bottom backs a seeded library up to an in-process S3
+bucket, wipes it, restores it, and compares every file."""
+import hashlib
+import shutil
+import time
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import func, select
+
+from backend.app.models import Artist, BackupRun, ImageRecord, Post, PostAttachment
+from backend.app.services import backup_restore
+from backend.app.services.backup_s3 import ListPage, ObjectMissing, S3Error, S3Target, StoredObject
+from tests.fake_s3 import FakeS3
+
+pytestmark = pytest.mark.integration
+
+TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _file(root, rel, data):
+    p = root / rel
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_bytes(data)
+    return p
+
+
+def _image(db_sync, path, data, mime="image/jpeg"):
+    db_sync.add(ImageRecord(path=str(path), sha256=_sha(data), size_bytes=len(data),
+                            mime=mime, origin="imported_filesystem"))
+
+
+# ── dumps ───────────────────────────────────────────────────────────────────
+
+
+def test_dumps_are_listed_newest_first_and_other_keys_ignored(monkeypatch):
+    page = ListPage([
+        StoredObject("database/fabledcurator-20261101T034000Z.dump", 10, ""),
+        StoredObject("database/fabledcurator-20261107T134258Z.dump", 30, ""),
+        StoredObject("database/notes.txt", 1, ""),
+    ], None)
+    monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: page)
+    dumps = backup_restore.list_dumps(TARGET)
+    assert [d.ts for d in dumps] == ["20261107T134258Z", "20261101T034000Z"]
+    assert backup_restore.find_dump(TARGET, "latest").size_bytes == 30
+    assert backup_restore.find_dump(TARGET, dumps[1].key).ts == "20261101T034000Z"
+    with pytest.raises(S3Error, match="not among"):
+        backup_restore.find_dump(TARGET, "database/fabledcurator-1.dump")
+
+
+def test_an_empty_bucket_has_no_dump_to_restore(monkeypatch):
+    monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: ListPage([], None))
+    with pytest.raises(S3Error, match="no database dumps"):
+        backup_restore.find_dump(TARGET, None)
+
+
+# ── one file ────────────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def fetches(monkeypatch):
+    got = []
+
+    def get_file(target, key, path, *, sha256=None, size=None, pacer=None):
+        got.append(key)
+        path.write_bytes(b"x" * size)
+    monkeypatch.setattr(backup_restore, "get_file", get_file)
+    return got
+
+
+def test_a_file_on_disk_at_its_size_is_left_alone(tmp_path, fetches):
+    p = _file(tmp_path, "a/one.jpg", b"xxx")
+    f = backup_restore.LibraryFile("a" * 64, (str(p),), 3)
+    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "present"
+    assert fetches == []
+
+
+def test_a_missing_file_is_fetched_by_its_key_to_its_path(tmp_path, fetches):
+    sha = "abcd" + "0" * 60
+    f = backup_restore.LibraryFile(sha, (str(tmp_path / "bea/one.JPG"),), 4)
+    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored"
+    assert fetches == [f"media/ab/cd/{sha}.jpg"]
+    assert (tmp_path / "bea/one.JPG").read_bytes() == b"xxxx"
+
+
+def test_a_recorded_key_is_used_over_the_computed_one(tmp_path, fetches):
+    f = backup_restore.LibraryFile("e" * 64, (str(tmp_path / "x.png"),), 2)
+    backup_restore.restore_file(TARGET, f, "media/ee/ee/recorded.png", tmp_path, None)
+    assert fetches == ["media/ee/ee/recorded.png"]
+
+
+def test_one_fetch_fills_every_path_that_shares_the_file(tmp_path, fetches):
+    paths = (str(tmp_path / "p1/set.zip"), str(tmp_path / "p2/set.zip"))
+    f = backup_restore.LibraryFile("c" * 64, paths, 5)
+    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored"
+    assert len(fetches) == 1
+    assert (tmp_path / "p2/set.zip").read_bytes() == b"xxxxx"
+    # With one copy left on disk, the other is copied, not fetched.
+    (tmp_path / "p1/set.zip").unlink()
+    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "copied"
+    assert len(fetches) == 1
+
+
+def test_a_path_outside_the_library_is_refused(tmp_path, fetches):
+    f = backup_restore.LibraryFile("d" * 64, ("/etc/passwd",), 3)
+    with pytest.raises(S3Error, match="outside"):
+        backup_restore.restore_file(TARGET, f, None, tmp_path, None)
+    f = backup_restore.LibraryFile("d" * 64, (str(tmp_path / "../escape.jpg"),), 3)
+    with pytest.raises(S3Error, match="outside"):
+        backup_restore.restore_file(TARGET, f, None, tmp_path, None)
+    assert fetches == []
+
+
+def test_the_work_list_gives_each_file_every_path(db_sync, tmp_path):
+    _image(db_sync, tmp_path / "a.jpg", b"one")
+    artist = Artist(name="Bea", slug="bea")
+    db_sync.add(artist)
+    db_sync.flush()
+    post = Post(artist_id=artist.id, external_post_id="p1")
+    db_sync.add(post)
+    db_sync.flush()
+    for i, post_id in enumerate((None, post.id)):
+        db_sync.add(PostAttachment(post_id=post_id, sha256=_sha(b"zip"),
+                                   path=str(tmp_path / f"{i}/set.zip"),
+                                   original_filename="set.zip", ext=".zip", size_bytes=3))
+    db_sync.commit()
+    files = {f.sha256: f for f in backup_restore.library_files(db_sync)}
+    assert files[_sha(b"zip")].paths == (str(tmp_path / "0/set.zip"), str(tmp_path / "1/set.zip"))
+    assert files[_sha(b"one")].paths == (str(tmp_path / "a.jpg"),)
+
+
+# ── the file phase as a run ─────────────────────────────────────────────────
+
+
+@pytest.fixture
+def run_env(monkeypatch, tmp_path):
+    from backend.app.tasks import backup as tasks
+    monkeypatch.setattr(tasks, "IMAGES_ROOT", tmp_path)
+    monkeypatch.setattr(
+        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
+    )
+    queued, thumbs = [], []
+    monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw))
+    monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay",
+                        lambda: thumbs.append(True))
+    return queued, thumbs
+
+
+def test_fetching_missing_files_counts_each_outcome(db_sync, tmp_path, run_env, monkeypatch):
+    from backend.app.tasks.backup import restore_offsite_task
+    _file(tmp_path, "here.jpg", b"here")
+    _image(db_sync, tmp_path / "here.jpg", b"here")
+    _image(db_sync, tmp_path / "gone.jpg", b"gone")
+    _image(db_sync, tmp_path / "lost.jpg", b"lost")
+    db_sync.commit()
+
+    def get_file(target, key, path, *, sha256=None, size=None, pacer=None):
+        if sha256 == _sha(b"lost"):
+            raise ObjectMissing(f"{key} is not in the curator bucket.")
+        path.write_bytes(b"gone")
+    monkeypatch.setattr(backup_restore, "get_file", get_file)
+
+    result = restore_offsite_task(files_only=True)
+    assert result["continues"] is False
+    row = db_sync.get(BackupRun, result["backup_run_id"])
+    db_sync.refresh(row)
+    m = row.manifest
+    assert (row.kind, row.status) == ("restore", "restored")
+    assert (m["restored"], m["present"], m["missing"], m["failed"]) == (1, 1, 1, 0)
+    assert "1 file(s) aren't in the bucket" in row.error
+    assert (tmp_path / "gone.jpg").read_bytes() == b"gone"
+    queued, thumbs = run_env
+    assert queued == [] and thumbs == [True]
+
+
+def test_a_restore_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
+    from backend.app.tasks import backup as tasks
+    _image(db_sync, tmp_path / "a.jpg", b"aa")
+    db_sync.commit()
+    monkeypatch.setattr(tasks, "RESTORE_CHUNK_SECONDS", -1)
+    result = tasks.restore_offsite_task(files_only=True)
+    queued, _ = run_env
+    assert result["continues"] is True
+    assert queued == [{"run_id": result["backup_run_id"]}]
+    row = db_sync.get(BackupRun, result["backup_run_id"])
+    db_sync.refresh(row)
+    assert row.status == "restoring" and row.last_progress_at is not None
+
+
+def test_a_restore_waits_for_a_backup_that_is_running(db_sync, run_env):
+    from backend.app.tasks.backup import restore_offsite_task
+    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
+                          started_at=datetime.now(UTC), manifest={}))
+    db_sync.commit()
+    assert restore_offsite_task(files_only=True)["skipped"] == "a media backup is running"
+
+
+def test_backups_wait_while_a_restore_is_going(db_sync, monkeypatch):
+    from backend.app.tasks.backup import backup_db_task, backup_media_task
+    db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                          started_at=datetime.now(UTC), manifest={}))
+    db_sync.commit()
+    assert backup_db_task(triggered_by="nightly")["skipped"] == "a restore is going"
+    assert backup_media_task(triggered_by="nightly")["skipped"] == "a restore is going"
+
+
+def test_a_long_restore_is_judged_stalled_by_its_heartbeat(db_sync):
+    from backend.app.tasks.maintenance import recover_stalled_backup_runs
+    long_ago = datetime.now(UTC) - timedelta(hours=20)
+    alive = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
+    silent = BackupRun(kind="restore", status="restoring", triggered_by="restore",
+                       started_at=long_ago, last_progress_at=long_ago, manifest={})
+    db_sync.add_all([alive, silent])
+    db_sync.commit()
+    assert recover_stalled_backup_runs() == 1
+    db_sync.refresh(alive)
+    db_sync.refresh(silent)
+    assert (alive.status, silent.status) == ("restoring", "error")
+
+
+# ── the round trip ──────────────────────────────────────────────────────────
+
+
+def test_a_library_comes_back_whole_from_the_bucket(db_sync, tmp_path, monkeypatch):
+    """Back the library up to a bucket, lose the files and change the
+    database, restore from the bucket: the rows are the backup's and every
+    file is back at its path with its bytes."""
+    from backend.app.services import backup_destination
+    from backend.app.tasks import backup as tasks
+
+    images = tmp_path / "images"
+    library = {
+        "bea/one.jpg": b"first picture",
+        "bea/two.png": b"second picture, a png",
+        "attachments/ab/set.zip": b"an archive of pictures" * 50,
+    }
+    for rel, data in library.items():
+        _file(images, rel, data)
+    _image(db_sync, images / "bea/one.jpg", library["bea/one.jpg"])
+    _image(db_sync, images / "bea/two.png", library["bea/two.png"], mime="image/png")
+    db_sync.add(PostAttachment(sha256=_sha(library["attachments/ab/set.zip"]),
+                               path=str(images / "attachments/ab/set.zip"),
+                               original_filename="set.zip", ext=".zip",
+                               size_bytes=len(library["attachments/ab/set.zip"])))
+    db_sync.commit()
+
+    monkeypatch.setattr(tasks, "IMAGES_ROOT", images)
+    monkeypatch.setattr(backup_destination, "seal", lambda secret: f"sealed:{secret}")
+    queued = []
+    monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw))
+    monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay", lambda: None)
+
+    with FakeS3() as s3:
+        monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target())
+        dumped = tasks.backup_db_task(triggered_by="manual")
+        tasks.backup_media_task(triggered_by="manual")
+        dump_key = [k for k in s3.objects if k.startswith("database/")][0]
+        assert len([k for k in s3.objects if k.startswith("media/")]) == 3
+
+        # Lose the files, and change the database after the dump.
+        shutil.rmtree(images)
+        db_sync.execute(ImageRecord.__table__.delete().where(
+            ImageRecord.sha256 == _sha(library["bea/two.png"])))
+        db_sync.add(Artist(name="Added later", slug="added-later"))
+        db_sync.commit()
+        time.sleep(1.1)   # the safety dump gets its own second-resolution name
+
+        first = tasks.restore_offsite_task(dump_key=dump_key)
+        assert queued == [{"run_id": first["backup_run_id"]}]
+        done = tasks.restore_offsite_task(run_id=first["backup_run_id"])
+        assert done["continues"] is False
+        dumps_now = [k for k in s3.objects if k.startswith("database/")]
+
+    # The database is the dump's: a fresh session, since the restore ended
+    # every connection that was open.
+    with tasks._sync_session_factory()() as s:
+        assert s.execute(select(func.count()).select_from(ImageRecord)).scalar() == 2
+        assert s.execute(select(Artist).where(Artist.slug == "added-later")).first() is None
+        row = s.get(BackupRun, done["backup_run_id"])
+        assert row.status == "restored" and row.error is None
+        assert row.manifest["restored"] == 3
+        assert row.manifest["dump"]["key"] == dump_key
+        # The database it replaced was saved to the bucket first.
+        assert row.manifest["saved_before"] in dumps_now and len(dumps_now) == 2
+        assert s.get(BackupRun, dumped["backup_run_id"]).status == "ok"
+        settings = s.execute(select(tasks.ImportSettings)).scalars().first()
+        assert settings.backup_s3_secret_sealed == "sealed:SK"
+    for rel, data in library.items():
+        assert (images / rel).read_bytes() == data

tests/test_backup_s3.py

@@ -212,6 +212,41 @@ def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
     assert not (tmp_path / "b.dump.partial").exists()
 
 
+def test_a_missing_object_is_named_as_missing(tmp_path):
+    body = b"<Error><Code>NoSuchKey</Code></Error>"
+    with _Server(lambda h: (404, body)) as srv, pytest.raises(s3.ObjectMissing):
+        s3.get_file(srv.target(), "media/ab/cd/x.jpg", tmp_path / "x.jpg")
+
+
+def test_a_fetched_file_is_kept_only_when_its_fingerprint_matches(tmp_path):
+    good = b"picture bytes"
+    sha = hashlib.sha256(good).hexdigest()
+    with _Server(lambda h: (200, good)) as srv:
+        s3.get_file(srv.target(), "k", tmp_path / "ok.jpg", sha256=sha, size=len(good))
+    assert (tmp_path / "ok.jpg").read_bytes() == good
+
+    with _Server(lambda h: (200, b"other bytes!!")) as srv, \
+            pytest.raises(s3.S3Error, match="different fingerprint"):
+        s3.get_file(srv.target(), "k", tmp_path / "bad.jpg", sha256=sha, size=len(good))
+    assert not (tmp_path / "bad.jpg").exists()
+    assert not (tmp_path / "bad.jpg.partial").exists()
+
+
+def test_a_short_download_is_not_kept(tmp_path):
+    with _Server(lambda h: (200, b"abc")) as srv, pytest.raises(s3.S3Error, match="3 bytes"):
+        s3.get_file(srv.target(), "k", tmp_path / "x", size=10)
+    assert not (tmp_path / "x").exists()
+
+
+def test_a_paced_download_counts_its_bytes(tmp_path):
+    pacer = s3.Pacer(None)
+    taken = []
+    pacer.take = taken.append
+    with _Server(lambda h: (200, b"z" * 2500)) as srv:
+        s3.get_file(srv.target(), "k", tmp_path / "z", pacer=pacer)
+    assert sum(taken) == 2500
+
+
 # ── Multipart ───────────────────────────────────────────────────────────────
 
 _STARTED = b"<InitiateMultipartUploadResult><UploadId>up1</UploadId></InitiateMultipartUploadResult>"

tests/test_backup_service.py

@@ -172,11 +172,33 @@ def test_restore_db_drops_schema_then_loads(tmp_path, fake_subprocess):
     assert fake_subprocess[0][0] == "psql"
     assert "-c" in fake_subprocess[0]
     assert "DROP SCHEMA IF EXISTS public CASCADE" in fake_subprocess[0][-1]
-    assert fake_subprocess[1][0] == "pg_restore"
+    assert "pg_terminate_backend" in fake_subprocess[0][-1]
+    assert "pg_restore" in fake_subprocess[1] and "--exit-on-error" in fake_subprocess[1]
     assert "-d" in fake_subprocess[1]
     assert str(dump_path) in fake_subprocess[1]
 
 
+def test_a_failed_restore_says_what_pg_restore_said(tmp_path, monkeypatch):
+    class _Done:
+        def __init__(self, code):
+            self.returncode = code
+
+    def run(cmd, stderr=None, **kw):
+        if "pg_restore" in cmd:
+            stderr.write(b"pg_restore: processing data\npg_restore: error: disk full\n")
+            return _Done(1)
+        return _Done(0)
+
+    monkeypatch.setattr("subprocess.run", run)
+    with pytest.raises(backup_service.RestoreFailed, match="error: disk full"):
+        backup_service.restore_db(db_url="postgresql://u@h/d", sql_path=tmp_path / "x.dump")
+
+
+def test_every_dump_leaves_out_account_logins(tmp_path, fake_subprocess):
+    backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
+    assert "--exclude-table-data=credential" in fake_subprocess[0]
+
+
 # --- restore_images --------------------------------------------------