FabledCurator/tests/test_backup_restore.py

"""Restoring from the off-site bucket (services/backup_restore.py and
restore_offsite_task): the database from a dated dump, then every library
file back at the path its row records, checked against its sha256 on arrival.
The round trip at the bottom backs a seeded library up to an in-process S3
bucket, wipes it, restores it, and compares every file."""
import hashlib
import shutil
import time
from datetime import UTC, datetime, timedelta

import pytest
from sqlalchemy import func, select

from backend.app.models import Artist, BackupRun, ImageRecord, Post, PostAttachment
from backend.app.services import backup_restore, backup_verify
from backend.app.services.backup_s3 import ListPage, ObjectMissing, S3Error, S3Target, StoredObject
from tests.fake_s3 import FakeS3

pytestmark = pytest.mark.integration

TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")


def _sha(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def _file(root, rel, data):
    p = root / rel
    p.parent.mkdir(parents=True, exist_ok=True)
    p.write_bytes(data)
    return p


def _image(db_sync, path, data, mime="image/jpeg"):
    db_sync.add(ImageRecord(path=str(path), sha256=_sha(data), size_bytes=len(data),
                            mime=mime, origin="imported_filesystem"))


# ── dumps ───────────────────────────────────────────────────────────────────


def test_dumps_are_listed_newest_first_and_other_keys_ignored(monkeypatch):
    page = ListPage([
        StoredObject("database/fabledcurator-20261101T034000Z.dump", 10, ""),
        StoredObject("database/fabledcurator-20261107T134258Z.dump", 30, ""),
        StoredObject("database/fabledcurator-20261107T134258Z.counts.json", 2, ""),
        StoredObject("database/notes.txt", 1, ""),
    ], None)
    monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: page)
    dumps = backup_restore.list_dumps(TARGET)
    assert [d.ts for d in dumps] == ["20261107T134258Z", "20261101T034000Z"]
    assert [d.counted for d in dumps] == [True, False]
    assert backup_restore.find_dump(TARGET, "latest").size_bytes == 30
    assert backup_restore.find_dump(TARGET, dumps[1].key).ts == "20261101T034000Z"
    with pytest.raises(S3Error, match="not among"):
        backup_restore.find_dump(TARGET, "database/fabledcurator-1.dump")


def test_an_empty_bucket_has_no_dump_to_restore(monkeypatch):
    monkeypatch.setattr(backup_restore, "list_objects", lambda *a, **kw: ListPage([], None))
    with pytest.raises(S3Error, match="no database dumps"):
        backup_restore.find_dump(TARGET, None)


# ── one file ────────────────────────────────────────────────────────────────


@pytest.fixture
def fetches(monkeypatch):
    got = []

    def get_file(target, key, path, *, sha256=None, size=None, pacer=None):
        got.append(key)
        path.write_bytes(b"x" * size)
    monkeypatch.setattr(backup_restore, "get_file", get_file)
    return got


def test_a_file_on_disk_at_its_size_is_left_alone(tmp_path, fetches):
    p = _file(tmp_path, "a/one.jpg", b"xxx")
    f = backup_restore.LibraryFile("a" * 64, (str(p),), 3)
    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "present"
    assert fetches == []


def test_a_missing_file_is_fetched_by_its_key_to_its_path(tmp_path, fetches):
    sha = "abcd" + "0" * 60
    f = backup_restore.LibraryFile(sha, (str(tmp_path / "bea/one.JPG"),), 4)
    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored"
    assert fetches == [f"media/ab/cd/{sha}.jpg"]
    assert (tmp_path / "bea/one.JPG").read_bytes() == b"xxxx"


def test_a_recorded_key_is_used_over_the_computed_one(tmp_path, fetches):
    f = backup_restore.LibraryFile("e" * 64, (str(tmp_path / "x.png"),), 2)
    backup_restore.restore_file(TARGET, f, "media/ee/ee/recorded.png", tmp_path, None)
    assert fetches == ["media/ee/ee/recorded.png"]


def test_one_fetch_fills_every_path_that_shares_the_file(tmp_path, fetches):
    paths = (str(tmp_path / "p1/set.zip"), str(tmp_path / "p2/set.zip"))
    f = backup_restore.LibraryFile("c" * 64, paths, 5)
    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "restored"
    assert len(fetches) == 1
    assert (tmp_path / "p2/set.zip").read_bytes() == b"xxxxx"
    # With one copy left on disk, the other is copied, not fetched.
    (tmp_path / "p1/set.zip").unlink()
    assert backup_restore.restore_file(TARGET, f, None, tmp_path, None) == "copied"
    assert len(fetches) == 1


def test_a_path_outside_the_library_is_refused(tmp_path, fetches):
    f = backup_restore.LibraryFile("d" * 64, ("/etc/passwd",), 3)
    with pytest.raises(S3Error, match="outside"):
        backup_restore.restore_file(TARGET, f, None, tmp_path, None)
    f = backup_restore.LibraryFile("d" * 64, (str(tmp_path / "../escape.jpg"),), 3)
    with pytest.raises(S3Error, match="outside"):
        backup_restore.restore_file(TARGET, f, None, tmp_path, None)
    assert fetches == []


def test_the_work_list_gives_each_file_every_path(db_sync, tmp_path):
    _image(db_sync, tmp_path / "a.jpg", b"one")
    artist = Artist(name="Bea", slug="bea")
    db_sync.add(artist)
    db_sync.flush()
    post = Post(artist_id=artist.id, external_post_id="p1")
    db_sync.add(post)
    db_sync.flush()
    for i, post_id in enumerate((None, post.id)):
        db_sync.add(PostAttachment(post_id=post_id, sha256=_sha(b"zip"),
                                   path=str(tmp_path / f"{i}/set.zip"),
                                   original_filename="set.zip", ext=".zip", size_bytes=3))
    db_sync.commit()
    files = {f.sha256: f for f in backup_restore.library_files(db_sync)}
    assert files[_sha(b"zip")].paths == (str(tmp_path / "0/set.zip"), str(tmp_path / "1/set.zip"))
    assert files[_sha(b"one")].paths == (str(tmp_path / "a.jpg"),)


# ── the file phase as a run ─────────────────────────────────────────────────


@pytest.fixture
def run_env(monkeypatch, tmp_path):
    from backend.app.tasks import backup as tasks
    monkeypatch.setattr(tasks, "IMAGES_ROOT", tmp_path)
    monkeypatch.setattr(
        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
    )
    queued, thumbs = [], []
    monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw))
    monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay",
                        lambda: thumbs.append(True))
    return queued, thumbs


def test_fetching_missing_files_counts_each_outcome(db_sync, tmp_path, run_env, monkeypatch):
    from backend.app.tasks.backup import restore_offsite_task
    _file(tmp_path, "here.jpg", b"here")
    _image(db_sync, tmp_path / "here.jpg", b"here")
    _image(db_sync, tmp_path / "gone.jpg", b"gone")
    _image(db_sync, tmp_path / "lost.jpg", b"lost")
    db_sync.commit()

    def get_file(target, key, path, *, sha256=None, size=None, pacer=None):
        if sha256 == _sha(b"lost"):
            raise ObjectMissing(f"{key} is not in the curator bucket.")
        path.write_bytes(b"gone")
    monkeypatch.setattr(backup_restore, "get_file", get_file)

    result = restore_offsite_task(files_only=True)
    assert result["continues"] is False
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    m = row.manifest
    assert (row.kind, row.status) == ("restore", "restored")
    assert (m["restored"], m["present"], m["missing"], m["failed"]) == (1, 1, 1, 0)
    assert "1 file(s) aren't in the bucket" in row.error
    assert (tmp_path / "gone.jpg").read_bytes() == b"gone"
    queued, thumbs = run_env
    assert queued == [] and thumbs == [True]


def test_a_restore_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
    from backend.app.tasks import backup as tasks
    _image(db_sync, tmp_path / "a.jpg", b"aa")
    db_sync.commit()
    monkeypatch.setattr(tasks, "RESTORE_CHUNK_SECONDS", -1)
    result = tasks.restore_offsite_task(files_only=True)
    queued, _ = run_env
    assert result["continues"] is True
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    assert row.status == "restoring" and row.last_progress_at is not None
    # The next chunk carries the token of the chunk that queued it.
    assert queued == [{"run_id": row.id, "after": row.manifest["chunk"]}]


def test_a_restart_resumes_the_file_phase_and_drops_the_stale_chunk(
    db_sync, tmp_path, run_env,
):
    from backend.app.tasks.backup import restore_offsite_task, resume_offsite_runs
    _file(tmp_path, "a.jpg", b"aa")
    _image(db_sync, tmp_path / "a.jpg", b"aa")
    now = datetime.now(UTC)
    files = BackupRun(kind="restore", status="restoring", triggered_by="restore",
                      started_at=now, manifest={"phase": "files", "cursor": "",
                                                "failures": [], "chunk": "killed"})
    loading = BackupRun(kind="restore", status="restoring", triggered_by="restore",
                        started_at=now, manifest={"phase": "database"})
    db_sync.add_all([files, loading])
    db_sync.commit()
    queued, _ = run_env

    # A restore stopped while loading its database isn't resumed.
    resume_offsite_runs()
    assert queued == [{"run_id": files.id, "resume": True}]

    # The killed chunk's own message, delivered again, is dropped.
    assert "another chunk" in restore_offsite_task(run_id=files.id, after="earlier")["skipped"]
    done = restore_offsite_task(run_id=files.id, resume=True)
    assert done["continues"] is False
    db_sync.refresh(files)
    assert files.status == "restored" and files.manifest["present"] == 1


def test_a_restore_waits_for_a_backup_that_is_running(db_sync, run_env):
    from backend.app.tasks.backup import restore_offsite_task
    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
                          started_at=datetime.now(UTC), manifest={}))
    db_sync.commit()
    assert restore_offsite_task(files_only=True)["skipped"] == "a media backup is running"


def test_backups_wait_while_a_restore_is_going(db_sync, monkeypatch):
    from backend.app.tasks.backup import backup_db_task, backup_media_task
    db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore",
                          started_at=datetime.now(UTC), manifest={}))
    db_sync.commit()
    assert backup_db_task(triggered_by="nightly")["skipped"] == "a restore is going"
    assert backup_media_task(triggered_by="nightly")["skipped"] == "a restore is going"


def test_a_long_restore_is_judged_stalled_by_its_heartbeat(db_sync):
    from backend.app.tasks.maintenance import recover_stalled_backup_runs
    long_ago = datetime.now(UTC) - timedelta(hours=20)
    alive = BackupRun(kind="restore", status="restoring", triggered_by="restore",
                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
    silent = BackupRun(kind="restore", status="restoring", triggered_by="restore",
                       started_at=long_ago, last_progress_at=long_ago, manifest={})
    db_sync.add_all([alive, silent])
    db_sync.commit()
    assert recover_stalled_backup_runs() == 1
    db_sync.refresh(alive)
    db_sync.refresh(silent)
    assert (alive.status, silent.status) == ("restoring", "error")


# ── the round trip ──────────────────────────────────────────────────────────


def test_a_library_comes_back_whole_from_the_bucket(db_sync, tmp_path, monkeypatch):
    """Back the library up to a bucket, lose the files and change the
    database, restore from the bucket: the rows are the backup's and every
    file is back at its path with its bytes."""
    from backend.app.services import backup_destination
    from backend.app.tasks import backup as tasks

    images = tmp_path / "images"
    library = {
        "bea/one.jpg": b"first picture",
        "bea/two.png": b"second picture, a png",
        "attachments/ab/set.zip": b"an archive of pictures" * 50,
    }
    for rel, data in library.items():
        _file(images, rel, data)
    _image(db_sync, images / "bea/one.jpg", library["bea/one.jpg"])
    _image(db_sync, images / "bea/two.png", library["bea/two.png"], mime="image/png")
    db_sync.add(PostAttachment(sha256=_sha(library["attachments/ab/set.zip"]),
                               path=str(images / "attachments/ab/set.zip"),
                               original_filename="set.zip", ext=".zip",
                               size_bytes=len(library["attachments/ab/set.zip"])))
    db_sync.commit()

    monkeypatch.setattr(tasks, "IMAGES_ROOT", images)
    monkeypatch.setattr(backup_destination, "seal", lambda secret: f"sealed:{secret}")
    queued = []
    monkeypatch.setattr(tasks.restore_offsite_task, "delay", lambda **kw: queued.append(kw))
    monkeypatch.setattr("backend.app.tasks.thumbnail.backfill_thumbnails.delay", lambda: None)

    with FakeS3() as s3:
        monkeypatch.setattr(backup_destination, "saved_target", lambda settings: s3.target())
        dumped = tasks.backup_db_task(triggered_by="manual")
        tasks.backup_media_task(triggered_by="manual")
        dump_key = [k for k in s3.objects if k.endswith(".dump")][0]
        # The dump's row counts sit beside it, for a test restore.
        assert backup_verify.counts_key(dump_key) in s3.objects
        assert len([k for k in s3.objects if k.startswith("media/")]) == 3

        # Lose the files, and change the database after the dump.
        shutil.rmtree(images)
        db_sync.execute(ImageRecord.__table__.delete().where(
            ImageRecord.sha256 == _sha(library["bea/two.png"])))
        db_sync.add(Artist(name="Added later", slug="added-later"))
        db_sync.commit()
        time.sleep(1.1)   # the safety dump gets its own second-resolution name

        first = tasks.restore_offsite_task(dump_key=dump_key)
        assert queued == [{"run_id": first["backup_run_id"]}]
        done = tasks.restore_offsite_task(run_id=first["backup_run_id"])
        assert done["continues"] is False
        dumps_now = [k for k in s3.objects if k.endswith(".dump")]

    # The database is the dump's: a fresh session, since the restore ended
    # every connection that was open.
    with tasks._sync_session_factory()() as s:
        assert s.execute(select(func.count()).select_from(ImageRecord)).scalar() == 2
        assert s.execute(select(Artist).where(Artist.slug == "added-later")).first() is None
        row = s.get(BackupRun, done["backup_run_id"])
        assert row.status == "restored" and row.error is None
        assert row.manifest["restored"] == 3
        assert row.manifest["dump"]["key"] == dump_key
        # The database it replaced was saved to the bucket first.
        assert row.manifest["saved_before"] in dumps_now and len(dumps_now) == 2
        assert s.get(BackupRun, dumped["backup_run_id"]).status == "ok"
        settings = s.execute(select(tasks.ImportSettings)).scalars().first()
        assert settings.backup_s3_secret_sealed == "sealed:SK"
    for rel, data in library.items():
        assert (images / rel).read_bytes() == data