"""Restoring from the off-site bucket (services/backup_restore.py and restore_offsite_task): the database from a dated dump, then every library file back at the path its row records, checked against its sha256 on arrival. The round trip at the bottom backs a seeded library up to an in-process S3 bucket, wipes it, restores it, and compares every file.""" import hashlib import shutil import time from datetime import UTC, datetime, timedelta import pytest from sqlalchemy import func, select from backend.app.models import Artist, BackupRun, ImageRecord, Post, PostAttachment from backend.app.services import backup_restore from backend.app.services.backup_s3 import ListPage, ObjectMissing, S3Error, S3Target, StoredObject from tests.fake_s3 import FakeS3 pytestmark = pytest.mark.integration TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK") def _sha(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def _file(root, rel, data): p = root / rel p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) return p def _image(db_sync, path, data, mime="image/jpeg"): db_sync.add(ImageRecord(path=str(path), sha256=_sha(data), size_bytes=len(data), mime=mime, origin="imported_filesystem")) # ── dumps ─────────────────────────────────────────────────────────────────── def test_dumps_are_listed_newest_first_and_other_keys_ignored(monkeypatch): page = ListPage([ StoredObject("database/fabledcurator-20261101T034000Z.dump", 10, ""), StoredObject("database/fabledcurator-20261107T134258Z.dump", 30, ""), StoredObject("database/notes.txt", 1, ""), ], None) monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: page) dumps = backup_restore.list_dumps(TARGET) assert [d.ts for d in dumps] == ["20261107T134258Z", "20261101T034000Z"] assert backup_restore.find_dump(TARGET, "latest").size_bytes == 30 assert backup_restore.find_dump(TARGET, dumps[1].key).ts == "20261101T034000Z" with pytest.raises(S3Error, match="not among"): backup_restore.find_dump(TARGET, "database/fabledcurator-1.dump") def test_an_empty_bucket_has_no_dump_to_restore(monkeypatch): monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: ListPage([], None)) with pytest.raises(S3Error, match="no database dumps"): backup_restore.find_dump(TARGET, None) # ── one file ──────────────────────────────────────────────────────────────── @pytest.fixture def fetches(monkeypatch): got = [] def get_file(target, key, path, *, sha256=None, size=None, pacer=None): got.append(key) path.write_bytes(b"x" * size) monkeypatch.setattr(backup_restore, "get_file", get_file) return got def test_a_file_on_disk_at_its_size_is_left_alone(tmp_path, fetches): p = _file(tmp_path, "a/one.jpg", b"xxx") f = backup_restore.LibraryFile("a" * 64, (str(p),), 3) assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "present" assert fetches == [] def test_a_missing_file_is_fetched_by_its_key_to_its_path(tmp_path, fetches): sha = "abcd" + "0" * 60 f = backup_restore.LibraryFile(sha, (str(tmp_path / "bea/one.JPG"),), 4) assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored" assert fetches == [f"media/ab/cd/{sha}.jpg"] assert (tmp_path / "bea/one.JPG").read_bytes() == b"xxxx" def test_a_recorded_key_is_used_over_the_computed_one(tmp_path, fetches): f = backup_restore.LibraryFile("e" * 64, (str(tmp_path / "x.png"),), 2) backup_restore.restore_file(TARGET, f, "media/ee/ee/recorded.png", tmp_path, None) assert fetches == ["media/ee/ee/recorded.png"] def test_one_fetch_fills_every_path_that_shares_the_file(tmp_path, fetches): paths = (str(tmp_path / "p1/set.zip"), str(tmp_path / "p2/set.zip")) f = backup_restore.LibraryFile("c" * 64, paths, 5) assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored" assert len(fetches) == 1 assert (tmp_path / "p2/set.zip").read_bytes() == b"xxxxx" # With one copy left on disk, the other is copied, not fetched. (tmp_path / "p1/set.zip").unlink() assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "copied" assert len(fetches) == 1 def test_a_path_outside_the_library_is_refused(tmp_path, fetches): f = backup_restore.LibraryFile("d" * 64, ("/etc/passwd",), 3) with pytest.raises(S3Error, match="outside"): backup_restore.restore_file(TARGET, f, None, tmp_path, None) f = backup_restore.LibraryFile("d" * 64, (str(tmp_path / "../escape.jpg"),), 3) with pytest.raises(S3Error, match="outside"): backup_restore.restore_file(TARGET, f, None, tmp_path, None) assert fetches == [] def test_the_work_list_gives_each_file_every_path(db_sync, tmp_path): _image(db_sync, tmp_path / "a.jpg", b"one") artist = Artist(name="Bea", slug="bea") db_sync.add(artist) db_sync.flush() post = Post(artist_id=artist.id, external_post_id="p1") db_sync.add(post) db_sync.flush() for i, post_id in enumerate((None, post.id)): db_sync.add(PostAttachment(post_id=post_id, sha256=_sha(b"zip"), path=str(tmp_path / f"{i}/set.zip"), original_filename="set.zip", ext=".zip", size_bytes=3)) db_sync.commit() files = {f.sha256: f for f in backup_restore.library_files(db_sync)} assert files[_sha(b"zip")].paths == (str(tmp_path / "0/set.zip"), str(tmp_path / "1/set.zip")) assert files[_sha(b"one")].paths == (str(tmp_path / "a.jpg"),) # ── the file phase as a run ───────────────────────────────────────────────── @pytest.fixture def run_env(monkeypatch, tmp_path): from backend.app.tasks import backup as tasks monkeypatch.setattr(tasks, "IMAGES_ROOT", tmp_path) monkeypatch.setattr( "backend.app.services.backup_destination.saved_target", lambda settings: TARGET, ) queued, thumbs = [], [] monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw)) monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay", lambda: thumbs.append(True)) return queued, thumbs def test_fetching_missing_files_counts_each_outcome(db_sync, tmp_path, run_env, monkeypatch): from backend.app.tasks.backup import restore_offsite_task _file(tmp_path, "here.jpg", b"here") _image(db_sync, tmp_path / "here.jpg", b"here") _image(db_sync, tmp_path / "gone.jpg", b"gone") _image(db_sync, tmp_path / "lost.jpg", b"lost") db_sync.commit() def get_file(target, key, path, *, sha256=None, size=None, pacer=None): if sha256 == _sha(b"lost"): raise ObjectMissing(f"{key} is not in the curator bucket.") path.write_bytes(b"gone") monkeypatch.setattr(backup_restore, "get_file", get_file) result = restore_offsite_task(files_only=True) assert result["continues"] is False row = db_sync.get(BackupRun, result["backup_run_id"]) db_sync.refresh(row) m = row.manifest assert (row.kind, row.status) == ("restore", "restored") assert (m["restored"], m["present"], m["missing"], m["failed"]) == (1, 1, 1, 0) assert "1 file(s) aren't in the bucket" in row.error assert (tmp_path / "gone.jpg").read_bytes() == b"gone" queued, thumbs = run_env assert queued == [] and thumbs == [True] def test_a_restore_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch): from backend.app.tasks import backup as tasks _image(db_sync, tmp_path / "a.jpg", b"aa") db_sync.commit() monkeypatch.setattr(tasks, "RESTORE_CHUNK_SECONDS", -1) result = tasks.restore_offsite_task(files_only=True) queued, _ = run_env assert result["continues"] is True row = db_sync.get(BackupRun, result["backup_run_id"]) db_sync.refresh(row) assert row.status == "restoring" and row.last_progress_at is not None # The next chunk carries the token of the chunk that queued it. assert queued == [{"run_id": row.id, "after": row.manifest["chunk"]}] def test_a_restart_resumes_the_file_phase_and_drops_the_stale_chunk( db_sync, tmp_path, run_env, ): from backend.app.tasks.backup import restore_offsite_task, resume_offsite_runs _file(tmp_path, "a.jpg", b"aa") _image(db_sync, tmp_path / "a.jpg", b"aa") now = datetime.now(UTC) files = BackupRun(kind="restore", status="restoring", triggered_by="restore", started_at=now, manifest={"phase": "files", "cursor": "", "failures": [], "chunk": "killed"}) loading = BackupRun(kind="restore", status="restoring", triggered_by="restore", started_at=now, manifest={"phase": "database"}) db_sync.add_all([files, loading]) db_sync.commit() queued, _ = run_env # A restore stopped while loading its database isn't resumed. resume_offsite_runs() assert queued == [{"run_id": files.id, "resume": True}] # The killed chunk's own message, delivered again, is dropped. assert "another chunk" in restore_offsite_task(run_id=files.id, after="earlier")["skipped"] done = restore_offsite_task(run_id=files.id, resume=True) assert done["continues"] is False db_sync.refresh(files) assert files.status == "restored" and files.manifest["present"] == 1 def test_a_restore_waits_for_a_backup_that_is_running(db_sync, run_env): from backend.app.tasks.backup import restore_offsite_task db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly", started_at=datetime.now(UTC), manifest={})) db_sync.commit() assert restore_offsite_task(files_only=True)["skipped"] == "a media backup is running" def test_backups_wait_while_a_restore_is_going(db_sync, monkeypatch): from backend.app.tasks.backup import backup_db_task, backup_media_task db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore", started_at=datetime.now(UTC), manifest={})) db_sync.commit() assert backup_db_task(triggered_by="nightly")["skipped"] == "a restore is going" assert backup_media_task(triggered_by="nightly")["skipped"] == "a restore is going" def test_a_long_restore_is_judged_stalled_by_its_heartbeat(db_sync): from backend.app.tasks.maintenance import recover_stalled_backup_runs long_ago = datetime.now(UTC) - timedelta(hours=20) alive = BackupRun(kind="restore", status="restoring", triggered_by="restore", started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={}) silent = BackupRun(kind="restore", status="restoring", triggered_by="restore", started_at=long_ago, last_progress_at=long_ago, manifest={}) db_sync.add_all([alive, silent]) db_sync.commit() assert recover_stalled_backup_runs() == 1 db_sync.refresh(alive) db_sync.refresh(silent) assert (alive.status, silent.status) == ("restoring", "error") # ── the round trip ────────────────────────────────────────────────────────── def test_a_library_comes_back_whole_from_the_bucket(db_sync, tmp_path, monkeypatch): """Back the library up to a bucket, lose the files and change the database, restore from the bucket: the rows are the backup's and every file is back at its path with its bytes.""" from backend.app.services import backup_destination from backend.app.tasks import backup as tasks images = tmp_path / "images" library = { "bea/one.jpg": b"first picture", "bea/two.png": b"second picture, a png", "attachments/ab/set.zip": b"an archive of pictures" * 50, } for rel, data in library.items(): _file(images, rel, data) _image(db_sync, images / "bea/one.jpg", library["bea/one.jpg"]) _image(db_sync, images / "bea/two.png", library["bea/two.png"], mime="image/png") db_sync.add(PostAttachment(sha256=_sha(library["attachments/ab/set.zip"]), path=str(images / "attachments/ab/set.zip"), original_filename="set.zip", ext=".zip", size_bytes=len(library["attachments/ab/set.zip"]))) db_sync.commit() monkeypatch.setattr(tasks, "IMAGES_ROOT", images) monkeypatch.setattr(backup_destination, "seal", lambda secret: f"sealed:{secret}") queued = [] monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw)) monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay", lambda: None) with FakeS3() as s3: monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target()) dumped = tasks.backup_db_task(triggered_by="manual") tasks.backup_media_task(triggered_by="manual") dump_key = [k for k in s3.objects if k.startswith("database/")][0] assert len([k for k in s3.objects if k.startswith("media/")]) == 3 # Lose the files, and change the database after the dump. shutil.rmtree(images) db_sync.execute(ImageRecord.__table__.delete().where( ImageRecord.sha256 == _sha(library["bea/two.png"]))) db_sync.add(Artist(name="Added later", slug="added-later")) db_sync.commit() time.sleep(1.1) # the safety dump gets its own second-resolution name first = tasks.restore_offsite_task(dump_key=dump_key) assert queued == [{"run_id": first["backup_run_id"]}] done = tasks.restore_offsite_task(run_id=first["backup_run_id"]) assert done["continues"] is False dumps_now = [k for k in s3.objects if k.startswith("database/")] # The database is the dump's: a fresh session, since the restore ended # every connection that was open. with tasks._sync_session_factory()() as s: assert s.execute(select(func.count()).select_from(ImageRecord)).scalar() == 2 assert s.execute(select(Artist).where(Artist.slug == "added-later")).first() is None row = s.get(BackupRun, done["backup_run_id"]) assert row.status == "restored" and row.error is None assert row.manifest["restored"] == 3 assert row.manifest["dump"]["key"] == dump_key # The database it replaced was saved to the bucket first. assert row.manifest["saved_before"] in dumps_now and len(dumps_now) == 2 assert s.get(BackupRun, dumped["backup_run_id"]).status == "ok" settings = s.execute(select(tasks.ImportSettings)).scalars().first() assert settings.backup_s3_secret_sealed == "sealed:SK" for rel, data in library.items(): assert (images / rel).read_bytes() == data