FabledCurator/tests/test_backup_media.py

"""The off-site copy of the library's files (services/backup_media.py and the
backup_media_task chunk loop): each image and attachment goes to the bucket
once, by sha256, the work list is what isn't there yet, and a file that can't
be sent is reported and tried again rather than stopping the run."""
from datetime import UTC, datetime, timedelta

import pytest
from sqlalchemy import select

from backend.app.models import (
    Artist,
    BackupBlob,
    BackupRun,
    ImageRecord,
    Post,
    PostAttachment,
)
from backend.app.services import backup_media
from backend.app.services.backup_s3 import ListPage, S3Error, S3Target, StoredObject

pytestmark = pytest.mark.integration

TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK")
TID = "http://archives.example:8081/curator"


def _sha(c: str) -> str:
    return c * 64


def _image(db_sync, tmp_path, c, body=b"img", ext=".jpg"):
    path = tmp_path / f"{c}{ext}"
    path.write_bytes(body)
    db_sync.add(ImageRecord(path=str(path), sha256=_sha(c), size_bytes=len(body),
                            mime="image/jpeg", origin="imported_filesystem"))
    return path


def _attachment(db_sync, tmp_path, c, body=b"zip", name="set.zip"):
    path = tmp_path / f"{c}-{name}"
    path.write_bytes(body)
    db_sync.add(PostAttachment(sha256=_sha(c), path=str(path), original_filename=name,
                               ext=".zip", size_bytes=len(body)))
    return path


# ── keys and the work list ──────────────────────────────────────────────────


def test_keys_are_content_addressed_under_media():
    sha = "abcd" + "0" * 60
    assert backup_media.blob_key(sha, "/images/x/Pic.JPG") == f"media/ab/cd/{sha}.jpg"
    assert backup_media.blob_key(sha, "/images/x/noext") == f"media/ab/cd/{sha}"
    assert backup_media.blob_key(sha, "/images/x/a.we ird") == f"media/ab/cd/{sha}"


def test_the_work_list_is_every_file_once_that_the_bucket_lacks(db_sync, tmp_path):
    _image(db_sync, tmp_path, "a")
    _image(db_sync, tmp_path, "b")
    _attachment(db_sync, tmp_path, "c")
    # The same archive held by a post as well is one file to send.
    artist = Artist(name="Bea", slug="bea")
    db_sync.add(artist)
    db_sync.flush()
    post = Post(artist_id=artist.id, external_post_id="p1")
    db_sync.add(post)
    db_sync.flush()
    db_sync.add(PostAttachment(post_id=post.id, sha256=_sha("c"),
                               path=str(tmp_path / "c-copy.zip"),
                               original_filename="set.zip", ext=".zip", size_bytes=3))
    db_sync.add(BackupBlob(target=TID, sha256=_sha("b"), key="media/bb", size_bytes=3))
    # A row for another bucket doesn't count for this one.
    db_sync.add(BackupBlob(target="http://other/bucket", sha256=_sha("a"), key="k",
                           size_bytes=3))
    db_sync.commit()

    shas = [b.sha256 for b in backup_media.pending(db_sync, TID)]
    assert shas == [_sha("a"), _sha("c")]
    assert [b.sha256 for b in backup_media.pending(db_sync, TID, after=_sha("a"))] == [_sha("c")]

    cov = backup_media.coverage(db_sync, TID)
    assert cov == {"files_total": 3, "bytes_total": 9, "files_sent": 1, "bytes_sent": 3}


# ── sending one file ────────────────────────────────────────────────────────


@pytest.fixture
def puts(monkeypatch):
    sent = []
    monkeypatch.setattr(backup_media, "put_file",
                        lambda t, key, path, **kw: sent.append(("whole", key, kw)))
    monkeypatch.setattr(backup_media, "put_file_in_parts",
                        lambda t, key, path, **kw: sent.append(("parts", key, kw)))
    return sent


def test_a_file_is_signed_with_its_recorded_sha(db_sync, tmp_path, puts):
    path = _image(db_sync, tmp_path, "a")
    blob = backup_media.Blob(_sha("a"), str(path), 3, "image/jpeg")
    key = backup_media.send(TARGET, blob, tmp_path, None)
    assert key == f"media/aa/aa/{_sha('a')}.jpg"
    (how, sent_key, kw) = puts[0]
    assert (how, sent_key) == ("whole", key)
    assert kw["payload_sha256"] == _sha("a") and kw["content_type"] == "image/jpeg"


def test_a_large_file_goes_in_parts(db_sync, tmp_path, puts, monkeypatch):
    monkeypatch.setattr(backup_media, "MULTIPART_OVER", 2)
    path = _attachment(db_sync, tmp_path, "c")
    backup_media.send(TARGET, backup_media.Blob(_sha("c"), str(path), 3, None), tmp_path, None)
    assert puts[0][0] == "parts"
    assert puts[0][2]["sha256"] == _sha("c")


def test_a_missing_or_changed_file_is_refused_before_sending(db_sync, tmp_path, puts):
    gone = backup_media.Blob(_sha("a"), str(tmp_path / "gone.jpg"), 3, None)
    with pytest.raises(S3Error, match="not on disk"):
        backup_media.send(TARGET, gone, tmp_path, None)
    path = _image(db_sync, tmp_path, "b", body=b"longer now")
    changed = backup_media.Blob(_sha("b"), str(path), 3, None)
    with pytest.raises(S3Error, match="when imported"):
        backup_media.send(TARGET, changed, tmp_path, None)
    assert puts == []


def test_a_refusal_for_an_object_already_there_counts_as_sent(db_sync, tmp_path, monkeypatch):
    def refused(*a, **kw):
        raise S3Error("HTTP 403 AccessDenied: append-only.")
    monkeypatch.setattr(backup_media, "put_file", refused)
    monkeypatch.setattr(backup_media, "object_present", lambda t, key, size: True)
    path = _image(db_sync, tmp_path, "a")
    key = backup_media.send(TARGET, backup_media.Blob(_sha("a"), str(path), 3, None),
                            tmp_path, None)
    assert key.startswith("media/aa/aa/")


# ── the run ─────────────────────────────────────────────────────────────────


@pytest.fixture
def run_env(monkeypatch, tmp_path):
    monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path)
    monkeypatch.setattr(
        "backend.app.services.backup_destination.saved_target", lambda settings: TARGET,
    )
    queued = []
    from backend.app.tasks import backup as tasks
    monkeypatch.setattr(tasks.backup_media_task, "delay", lambda **kw: queued.append(kw))
    return queued


def _send_all_but(bad_sha):
    def send(target, blob, images_root, pacer):
        if blob.sha256 == bad_sha:
            raise S3Error("HTTP 503: busy.")
        return backup_media.blob_key(blob.sha256, blob.path)
    return send


def test_a_run_sends_what_is_missing_and_reports_what_failed(
    db_sync, tmp_path, run_env, monkeypatch,
):
    from backend.app.tasks.backup import backup_media_task
    for c in "abc":
        _image(db_sync, tmp_path, c)
    db_sync.commit()
    monkeypatch.setattr(backup_media, "send", _send_all_but(_sha("b")))

    result = backup_media_task(triggered_by="manual")
    assert result["continues"] is False
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    assert (row.kind, row.status, row.size_bytes) == ("media", "ok", 6)
    assert row.manifest["sent"] == 2 and row.manifest["failed"] == 1
    assert row.manifest["offsite"]["bucket"] == "curator"
    assert "1 file(s) weren't sent" in row.error
    assert row.manifest["failures"][0]["error"] == "HTTP 503: busy."
    sent = set(db_sync.execute(select(BackupBlob.sha256)).scalars())
    assert sent == {_sha("a"), _sha("c")}

    # The next run tries only the file that failed.
    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
    again = backup_media_task(triggered_by="manual")
    row = db_sync.get(BackupRun, again["backup_run_id"])
    db_sync.refresh(row)
    assert row.manifest["sent"] == 1 and row.error is None


def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch):
    from backend.app.tasks import backup as tasks
    for c in "ab":
        _image(db_sync, tmp_path, c)
    db_sync.commit()
    monkeypatch.setattr(backup_media, "send", _send_all_but(None))
    monkeypatch.setattr(tasks, "MEDIA_CHUNK_SECONDS", -1)

    result = tasks.backup_media_task(triggered_by="manual")
    assert result["continues"] is True
    row = db_sync.get(BackupRun, result["backup_run_id"])
    db_sync.refresh(row)
    assert row.status == "running" and row.last_progress_at is not None
    # The next chunk carries the token of the chunk that queued it.
    assert run_env == [{"run_id": row.id, "after": row.manifest["chunk"]}]


def _going(db_sync, chunk):
    row = BackupRun(kind="media", status="running", triggered_by="manual",
                    started_at=datetime.now(UTC), size_bytes=0,
                    manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
                              "chunk": chunk})
    db_sync.add(row)
    db_sync.commit()
    return row


def test_only_the_chunk_queued_by_the_one_carrying_the_run_goes_on(
    db_sync, tmp_path, run_env, monkeypatch,
):
    from backend.app.tasks.backup import backup_media_task
    _image(db_sync, tmp_path, "a")
    row = _going(db_sync, "live")
    monkeypatch.setattr(backup_media, "send", _send_all_but(None))

    # A killed chunk's own message, delivered again, is dropped.
    stale = backup_media_task(run_id=row.id, after="earlier")
    assert "another chunk" in stale["skipped"]
    assert backup_media_task(run_id=row.id, after="live")["continues"] is False
    db_sync.refresh(row)
    assert row.status == "ok" and row.manifest["sent"] == 1


def test_a_resume_takes_the_run_and_the_chunk_it_replaced_stops(
    db_sync, tmp_path, run_env, monkeypatch,
):
    from backend.app.tasks import backup as tasks
    for c in "ab":
        _image(db_sync, tmp_path, c)
    row = _going(db_sync, "killed")

    def send_then_lose_the_run(target, blob, images_root, pacer):
        # While this chunk sends its first file, a newer one takes the run.
        with tasks._sync_session_factory()() as s:
            other = s.get(BackupRun, row.id)
            other.manifest = {**other.manifest, "chunk": "newer"}
            s.commit()
        return backup_media.blob_key(blob.sha256, blob.path)
    monkeypatch.setattr(backup_media, "send", send_then_lose_the_run)

    result = tasks.backup_media_task(run_id=row.id, resume=True)
    assert "took the run over" in result["skipped"]
    assert run_env == []
    db_sync.refresh(row)
    assert row.status == "running" and row.manifest["chunk"] == "newer"
    assert row.manifest["sent"] == 0 and row.manifest["cursor"] == ""


def test_a_restart_resumes_every_media_run_still_going(db_sync, run_env):
    from backend.app.tasks.backup import resume_offsite_runs
    going = _going(db_sync, "killed")
    db_sync.add(BackupRun(kind="media", status="ok", triggered_by="manual",
                          started_at=datetime.now(UTC), manifest={}))
    db_sync.commit()
    assert resume_offsite_runs()["media"] == [going.id]
    assert run_env == [{"run_id": going.id, "resume": True}]


def test_a_second_run_waits_for_the_one_going(db_sync, run_env):
    from backend.app.tasks.backup import backup_media_task
    now = datetime.now(UTC)
    db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly",
                          started_at=now, manifest={}))
    db_sync.commit()
    assert "already running" in backup_media_task(triggered_by="manual")["skipped"]


def test_without_a_destination_the_run_fails_with_why(db_sync, monkeypatch):
    from backend.app.tasks.backup import backup_media_task
    monkeypatch.setattr(
        "backend.app.services.backup_destination.saved_target", lambda settings: None,
    )
    with pytest.raises(S3Error):
        backup_media_task(triggered_by="manual")
    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "media")).scalar_one()
    assert row.status == "error" and "No off-site destination" in row.error


def test_a_long_media_run_is_judged_stalled_by_its_heartbeat(db_sync):
    from backend.app.tasks.maintenance import recover_stalled_backup_runs
    long_ago = datetime.now(UTC) - timedelta(hours=20)
    alive = BackupRun(kind="media", status="running", triggered_by="nightly",
                      started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={})
    silent = BackupRun(kind="media", status="running", triggered_by="nightly",
                       started_at=long_ago, last_progress_at=long_ago, manifest={})
    db_sync.add_all([alive, silent])
    db_sync.commit()
    assert recover_stalled_backup_runs() == 1
    db_sync.refresh(alive)
    db_sync.refresh(silent)
    assert (alive.status, silent.status) == ("running", "error")


# ── reconcile ───────────────────────────────────────────────────────────────


def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has(
    db_sync, tmp_path, monkeypatch,
):
    _image(db_sync, tmp_path, "a")            # row, but the bucket lost it
    _image(db_sync, tmp_path, "b")            # in the bucket, no row
    _image(db_sync, tmp_path, "c")            # neither
    key_a = backup_media.blob_key(_sha("a"), "x.jpg")
    key_b = backup_media.blob_key(_sha("b"), "x.jpg")
    db_sync.add(BackupBlob(target=TID, sha256=_sha("a"), key=key_a, size_bytes=3))
    db_sync.commit()
    pages = iter([
        ListPage([StoredObject("media/zz/other", 1, "")], "next"),
        ListPage([StoredObject(key_b, 3, "")], None),
    ])
    monkeypatch.setattr(backup_media, "list_objects", lambda *a, **kw: next(pages))
    monkeypatch.setattr(backup_media.time, "sleep", lambda s: None)

    assert backup_media.reconcile(db_sync, TARGET) == {
        "listed": 2, "dropped": 1, "adopted": 1,
    }
    assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}


def test_the_weekly_check_is_kept_as_a_run_with_its_counts(db_sync, run_env, monkeypatch):
    from backend.app.tasks import backup as tasks
    monkeypatch.setattr(backup_media, "reconcile",
                        lambda session, target: {"listed": 9, "dropped": 2, "adopted": 1})

    out = tasks.reconcile_media_backup()
    row = db_sync.get(BackupRun, out["backup_run_id"])
    db_sync.refresh(row)
    assert (row.kind, row.status, row.triggered_by) == ("reconcile", "ok", "weekly")
    assert row.finished_at is not None
    assert {k: row.manifest[k] for k in ("listed", "dropped", "adopted")} == {
        "listed": 9, "dropped": 2, "adopted": 1,
    }
    assert row.manifest["offsite"]["bucket"] == "curator"


def test_a_weekly_check_that_fails_is_kept_with_why(db_sync, run_env, monkeypatch):
    from backend.app.tasks import backup as tasks

    def unreachable(session, target):
        raise S3Error("HTTP 503: busy.")
    monkeypatch.setattr(backup_media, "reconcile", unreachable)

    with pytest.raises(S3Error):
        tasks.reconcile_media_backup()
    row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "reconcile")).scalar_one()
    assert row.status == "error"
    assert row.error == "S3Error: HTTP 503: busy."