FabledCurator/tests/test_backup_media.py

"""The off-site copy of the library's files (services/backup_media.py and the
backup_media_task chunk loop): each image and attachment goes to the bucket
once, by sha256, the work list is what isn't there yet, and a file that can't
be sent is reported and tried again rather than stopping the run."""
from datetime import UTC, datetime, timedelta

import pytest
from sqlalchemy import select

from backend.app.models import BackupBlob, BackupRun, ImageRecord, PostAttachment
from backend.app.services import backup_media
from backend.app.services.backup_s3 import ListPage, S3Error, S3Target, StoredObject

pytestmark = pytest.mark.integration

TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")
TID = "http://archives.example:8081/curator"


def _sha(c: str) -> str:
    return c * 64


def _image(db_sync, tmp_path, c, body=b"img", ext=".jpg"):
    path = tmp_path / f"{c}{ext}"
    path.write_bytes(body)
    db_sync.add(ImageRecord(path=str(path), sha256=_sha(c), size_bytes=len(body),
                            mime="image/jpeg", origin="imported_filesystem"))
    return path


def _attachment(db_sync, tmp_path, c, body=b"zip", name="set.zip"):
    path = tmp_path / f"{c}-{name}"
    path.write_bytes(body)
    db_sync.add(PostAttachment(sha256=_sha(c), path=str(path), original_filename=name,
                               ext=".zip", size_bytes=len(body)))
    return path


# ── keys and the work list ──────────────────────────────────────────────────


def test_keys_are_content_addressed_under_media():
    sha = "abcd" + "0" * 60
    assert backup_media.blob_key(sha, "/images/x/Pic.JPG") == f"media/ab/cd/{sha}.jpg"
    assert backup_media.blob_key(sha, "/images/x/noext") == f"media/ab/cd/{sha}"
    assert backup_media.blob_key(sha, "/images/x/a.we ird") == f"media/ab/cd/{sha}"


def test_the_work_list_is_every_file_once_that_the_bucket_lacks(db_sync, tmp_path):
    _image(db_sync, tmp_path, "a")
    _image(db_sync, tmp_path, "b")
    _attachment(db_sync, tmp_path, "c")
    # The same archive held by a second post is one file to send.
    db_sync.add(PostAttachment(sha256=_sha("c"), path=str(tmp_path / "c-copy.zip"),
                               original_filename="set.zip", ext=".zip", size_bytes=3))
    db_sync.add(BackupBlob(target=TID, sha256=_sha("b"), key="media/bb", size_bytes=3))
    # A row for another bucket doesn't count for this one.
    db_sync.add(BackupBlob(target="http://other/bucket", sha256=_sha("a"), key="k",
                           size_bytes=3))
    db_sync.commit()

    shas = [b.sha256 for b in backup_media.pending(db_sync, TID)]
    assert shas == [_sha("a"), _sha("c")]
    assert [b.sha256 for b in backup_media.pending(db_sync, TID, after=_sha("a"))] == [_sha("c")]

    cov = backup_media.coverage(db_sync, TID)
    assert cov == {"files_total": 3, "bytes_total": 9, "files_sent": 1, "bytes_sent": 3}


# ── sending one file ────────────────────────────────────────────────────────


@pytest.fixture
def puts(monkeypatch):
    sent = []
    monkeypatch.setattr(backup_media, "put_file",
                        lambda t, key, path, **kw: sent.append(("whole", key, kw)))
    monkeypatch.setattr(backup_media, "put_file_in_parts",
                        lambda t, key, path, **kw: sent.append(("parts", key, kw)))
    return sent


def test_a_file_is_signed_with_its_recorded_sha(db_sync, tmp_path, puts):
    path = _image(db_sync, tmp_path, "a")
    blob = backup_media.Blob(_sha("a"), str(path), 3, "image/jpeg")
    key = backup_media.send(TARGET, blob, tmp_path, None)
    assert key == f"media/aa/aa/{_sha('a')}.jpg"
    (how, sent_key, kw) = puts[0]
    assert (how, sent_key) == ("whole", key)
    assert kw["payload_sha256"] == _sha("a") and kw["content_type"] == "image/jpeg"


def test_a_large_file_goes_in_parts(db_sync, tmp_path, puts, monkeypatch):
    monkeypatch.setattr(backup_media, "MULTIPART_OVER", 2)
    path = _attachment(db_sync, tmp_path, "c")
    backup_media.send(TARGET, backup_media.Blob(_sha("c"), str(path), 3, None), tmp_path, None)
    assert puts[0][0] == "parts"
    assert puts[0][2]["sha256"] == _sha("c")


def test_a_missing_or_changed_file_is_refused_before_sending(db_sync, tmp_path, puts):
    gone = backup_media.Blob(_sha("a"), str(tmp_path / "gone.jpg"), 3, None)
    with pytest.raises(S3Error, match="not on disk"):
        backup_media.send(TARGET, gone, tmp_path, None)
    path = _image(db_sync, tmp_path, "b", body=b"longer now")
    changed = backup_media.Blob(_sha("b"), str(path), 3, None)
    with pytest.raises(S3Error, match="when imported"):
        backup_media.send(TARGET, changed, tmp_path, None)
    assert puts == []


def test_a_refusal_for_an_object_already_there_counts_as_sent(db_sync, tmp_path, monkeypatch):
    def refused(*a, **kw):
        raise S3Error("HTTP 403 AccessDenied: append-only.")
    monkeypatch.setattr(backup_media, "put_file", refused)
    monkeypatch.setattr(backup_media, "object_present", lambda t, key, size: True)
    path = _image(db_sync, tmp_path, "a")
    key = backup_media.send(TARGET, backup_media.Blob(_sha("a"), str(path), 3, None),
                            tmp_path, None)
    assert key.startswith("media/aa/aa/")


# ── the run ─────────────────────────────────────────────────────────────────


@pytest.fixture
def run_env(monkeypatch, tmp_path):
    monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path)
    monkeypatch.setattr(
        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
    )
    queued = []
    from backend.app.tasks import backup as tasks
    monkeypatch.setattr(tasks.backup_media_task, "delay", lambda **kw: queued.append(kw))
    return queued


def _send_all_but(bad_sha):
    def send(target, blob, images_root, pacer):
        if blob.sha256 == bad_sha:
            raise S3Error("HTTP 503: busy.")
        return backup_media.blob_key(blob.sha256, blob.path)
    return send


def test_a_run_sends_what_is_missing_and_reports_what_failed(
    db_sync, tmp_path, run_env, monkeypatch,
):
    from backend.app.tasks.backup import backup_media_task
    for c in "abc":
        _image(db_sync, tmp_path, c)
    db_sync.commit()
    monkeypatch.setattr(backup_media, "send", _send_all_but(_sha("b")))

    result = backup_media_task(triggered_by="manual")
    assert result["continues"] is False
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    assert (row.kind, row.status, row.size_bytes) == ("media", "ok", 6)
    assert row.manifest["sent"] == 2 and row.manifest["failed"] == 1
    assert row.manifest["offsite"]["bucket"] == "curator"
    assert "1 file(s) weren't sent" in row.error
    assert row.manifest["failures"][0]["error"] == "HTTP 503: busy."
    sent = set(db_sync.execute(select(BackupBlob.sha256)).scalars())
    assert sent == {_sha("a"), _sha("c")}

    # The next run tries only the file that failed.
    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
    again = backup_media_task(triggered_by="manual")
    row = db_sync.get(BackupRun, again["backup_run_id"])
    db_sync.refresh(row)
    assert row.manifest["sent"] == 1 and row.error is None


def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
    from backend.app.tasks import backup as tasks
    for c in "ab":
        _image(db_sync, tmp_path, c)
    db_sync.commit()
    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
    monkeypatch.setattr(tasks, "MEDIA_CHUNK_SECONDS", -1)

    result = tasks.backup_media_task(triggered_by="manual")
    assert result["continues"] is True
    assert run_env == [{"run_id": result["backup_run_id"]}]
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    assert row.status == "running" and row.last_progress_at is not None


def test_a_second_run_waits_for_the_one_going(db_sync, run_env):
    from backend.app.tasks.backup import backup_media_task
    now = datetime.now(UTC)
    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
                          started_at=now, manifest={}))
    db_sync.commit()
    assert "already running" in backup_media_task(triggered_by="manual")["skipped"]


def test_without_a_destination_the_run_fails_with_why(db_sync, monkeypatch):
    from backend.app.tasks.backup import backup_media_task
    monkeypatch.setattr(
        "backend.app.services.backup_destination.saved_target", lambda settings: None,
    )
    with pytest.raises(S3Error):
        backup_media_task(triggered_by="manual")
    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "media")).scalar_one()
    assert row.status == "error" and "No off-site destination" in row.error


def test_a_long_media_run_is_judged_stalled_by_its_heartbeat(db_sync):
    from backend.app.tasks.maintenance import recover_stalled_backup_runs
    long_ago = datetime.now(UTC) - timedelta(hours=20)
    alive = BackupRun(kind="media", status="running", triggered_by="nightly",
                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
    silent = BackupRun(kind="media", status="running", triggered_by="nightly",
                       started_at=long_ago, last_progress_at=long_ago, manifest={})
    db_sync.add_all([alive, silent])
    db_sync.commit()
    assert recover_stalled_backup_runs() == 1
    db_sync.refresh(alive)
    db_sync.refresh(silent)
    assert (alive.status, silent.status) == ("running", "error")


# ── reconcile ───────────────────────────────────────────────────────────────


def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
    db_sync, tmp_path, monkeypatch,
):
    _image(db_sync, tmp_path, "a")            # row, but the bucket lost it
    _image(db_sync, tmp_path, "b")            # in the bucket, no row
    _image(db_sync, tmp_path, "c")            # neither
    key_a = backup_media.blob_key(_sha("a"), "x.jpg")
    key_b = backup_media.blob_key(_sha("b"), "x.jpg")
    db_sync.add(BackupBlob(target=TID, sha256=_sha("a"), key=key_a, size_bytes=3))
    db_sync.commit()
    pages = iter([
        ListPage([StoredObject("media/zz/other", 1, "")], "next"),
        ListPage([StoredObject(key_b, 3, "")], None),
    ])
    monkeypatch.setattr(backup_media, "list_objects", lambda *a, **kw: next(pages))
    monkeypatch.setattr(backup_media.time, "sleep", lambda s: None)

    assert backup_media.reconcile(db_sync, TARGET) == {
        "listed": 2, "dropped": 1, "adopted": 1,
    }
    assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}