"""The off-site copy of the library's files (services/backup_media.py and the backup_media_task chunk loop): each image and attachment goes to the bucket once, by sha256, the work list is what isn't there yet, and a file that can't be sent is reported and tried again rather than stopping the run.""" from datetime import UTC, datetime, timedelta import pytest from sqlalchemy import select from backend.app.models import BackupBlob, BackupRun, ImageRecord, PostAttachment from backend.app.services import backup_media from backend.app.services.backup_s3 import ListPage, S3Error, S3Target, StoredObject pytestmark = pytest.mark.integration TARGET = S3Target("http://archives.example:8081", "curator", "AK", "SK") TID = "http://archives.example:8081/curator" def _sha(c: str) -> str: return c * 64 def _image(db_sync, tmp_path, c, body=b"img", ext=".jpg"): path = tmp_path / f"{c}{ext}" path.write_bytes(body) db_sync.add(ImageRecord(path=str(path), sha256=_sha(c), size_bytes=len(body), mime="image/jpeg", origin="imported_filesystem")) return path def _attachment(db_sync, tmp_path, c, body=b"zip", name="set.zip"): path = tmp_path / f"{c}-{name}" path.write_bytes(body) db_sync.add(PostAttachment(sha256=_sha(c), path=str(path), original_filename=name, ext=".zip", size_bytes=len(body))) return path # ── keys and the work list ────────────────────────────────────────────────── def test_keys_are_content_addressed_under_media(): sha = "abcd" + "0" * 60 assert backup_media.blob_key(sha, "/images/x/Pic.JPG") == f"media/ab/cd/{sha}.jpg" assert backup_media.blob_key(sha, "/images/x/noext") == f"media/ab/cd/{sha}" assert backup_media.blob_key(sha, "/images/x/a.we ird") == f"media/ab/cd/{sha}" def test_the_work_list_is_every_file_once_that_the_bucket_lacks(db_sync, tmp_path): _image(db_sync, tmp_path, "a") _image(db_sync, tmp_path, "b") _attachment(db_sync, tmp_path, "c") # The same archive held by a second post is one file to send. db_sync.add(PostAttachment(sha256=_sha("c"), path=str(tmp_path / "c-copy.zip"), original_filename="set.zip", ext=".zip", size_bytes=3)) db_sync.add(BackupBlob(target=TID, sha256=_sha("b"), key="media/bb", size_bytes=3)) # A row for another bucket doesn't count for this one. db_sync.add(BackupBlob(target="http://other/bucket", sha256=_sha("a"), key="k", size_bytes=3)) db_sync.commit() shas = [b.sha256 for b in backup_media.pending(db_sync, TID)] assert shas == [_sha("a"), _sha("c")] assert [b.sha256 for b in backup_media.pending(db_sync, TID, after=_sha("a"))] == [_sha("c")] cov = backup_media.coverage(db_sync, TID) assert cov == {"files_total": 3, "bytes_total": 9, "files_sent": 1, "bytes_sent": 3} # ── sending one file ──────────────────────────────────────────────────────── @pytest.fixture def puts(monkeypatch): sent = [] monkeypatch.setattr(backup_media, "put_file", lambda t, key, path, **kw: sent.append(("whole", key, kw))) monkeypatch.setattr(backup_media, "put_file_in_parts", lambda t, key, path, **kw: sent.append(("parts", key, kw))) return sent def test_a_file_is_signed_with_its_recorded_sha(db_sync, tmp_path, puts): path = _image(db_sync, tmp_path, "a") blob = backup_media.Blob(_sha("a"), str(path), 3, "image/jpeg") key = backup_media.send(TARGET, blob, tmp_path, None) assert key == f"media/aa/aa/{_sha('a')}.jpg" (how, sent_key, kw) = puts[0] assert (how, sent_key) == ("whole", key) assert kw["payload_sha256"] == _sha("a") and kw["content_type"] == "image/jpeg" def test_a_large_file_goes_in_parts(db_sync, tmp_path, puts, monkeypatch): monkeypatch.setattr(backup_media, "MULTIPART_OVER", 2) path = _attachment(db_sync, tmp_path, "c") backup_media.send(TARGET, backup_media.Blob(_sha("c"), str(path), 3, None), tmp_path, None) assert puts[0][0] == "parts" assert puts[0][2]["sha256"] == _sha("c") def test_a_missing_or_changed_file_is_refused_before_sending(db_sync, tmp_path, puts): gone = backup_media.Blob(_sha("a"), str(tmp_path / "gone.jpg"), 3, None) with pytest.raises(S3Error, match="not on disk"): backup_media.send(TARGET, gone, tmp_path, None) path = _image(db_sync, tmp_path, "b", body=b"longer now") changed = backup_media.Blob(_sha("b"), str(path), 3, None) with pytest.raises(S3Error, match="when imported"): backup_media.send(TARGET, changed, tmp_path, None) assert puts == [] def test_a_refusal_for_an_object_already_there_counts_as_sent(db_sync, tmp_path, monkeypatch): def refused(*a, **kw): raise S3Error("HTTP 403 AccessDenied: append-only.") monkeypatch.setattr(backup_media, "put_file", refused) monkeypatch.setattr(backup_media, "object_present", lambda t, key, size: True) path = _image(db_sync, tmp_path, "a") key = backup_media.send(TARGET, backup_media.Blob(_sha("a"), str(path), 3, None), tmp_path, None) assert key.startswith("media/aa/aa/") # ── the run ───────────────────────────────────────────────────────────────── @pytest.fixture def run_env(monkeypatch, tmp_path): monkeypatch.setattr("backend.app.tasks.backup.IMAGES_ROOT", tmp_path) monkeypatch.setattr( "backend.app.services.backup_destination.saved_target", lambda settings: TARGET, ) queued = [] from backend.app.tasks import backup as tasks monkeypatch.setattr(tasks.backup_media_task, "delay", lambda **kw: queued.append(kw)) return queued def _send_all_but(bad_sha): def send(target, blob, images_root, pacer): if blob.sha256 == bad_sha: raise S3Error("HTTP 503: busy.") return backup_media.blob_key(blob.sha256, blob.path) return send def test_a_run_sends_what_is_missing_and_reports_what_failed( db_sync, tmp_path, run_env, monkeypatch, ): from backend.app.tasks.backup import backup_media_task for c in "abc": _image(db_sync, tmp_path, c) db_sync.commit() monkeypatch.setattr(backup_media, "send", _send_all_but(_sha("b"))) result = backup_media_task(triggered_by="manual") assert result["continues"] is False row = db_sync.get(BackupRun, result["backup_run_id"]) db_sync.refresh(row) assert (row.kind, row.status, row.size_bytes) == ("media", "ok", 6) assert row.manifest["sent"] == 2 and row.manifest["failed"] == 1 assert row.manifest["offsite"]["bucket"] == "curator" assert "1 file(s) weren't sent" in row.error assert row.manifest["failures"][0]["error"] == "HTTP 503: busy." sent = set(db_sync.execute(select(BackupBlob.sha256)).scalars()) assert sent == {_sha("a"), _sha("c")} # The next run tries only the file that failed. monkeypatch.setattr(backup_media, "send", _send_all_but(None)) again = backup_media_task(triggered_by="manual") row = db_sync.get(BackupRun, again["backup_run_id"]) db_sync.refresh(row) assert row.manifest["sent"] == 1 and row.error is None def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, monkeypatch): from backend.app.tasks import backup as tasks for c in "ab": _image(db_sync, tmp_path, c) db_sync.commit() monkeypatch.setattr(backup_media, "send", _send_all_but(None)) monkeypatch.setattr(tasks, "MEDIA_CHUNK_SECONDS", -1) result = tasks.backup_media_task(triggered_by="manual") assert result["continues"] is True assert run_env == [{"run_id": result["backup_run_id"]}] row = db_sync.get(BackupRun, result["backup_run_id"]) db_sync.refresh(row) assert row.status == "running" and row.last_progress_at is not None def test_a_second_run_waits_for_the_one_going(db_sync, run_env): from backend.app.tasks.backup import backup_media_task now = datetime.now(UTC) db_sync.add(BackupRun(kind="media", status="running", triggered_by="nightly", started_at=now, manifest={})) db_sync.commit() assert "already running" in backup_media_task(triggered_by="manual")["skipped"] def test_without_a_destination_the_run_fails_with_why(db_sync, monkeypatch): from backend.app.tasks.backup import backup_media_task monkeypatch.setattr( "backend.app.services.backup_destination.saved_target", lambda settings: None, ) with pytest.raises(S3Error): backup_media_task(triggered_by="manual") row = db_sync.execute(select(BackupRun).where(BackupRun.kind == "media")).scalar_one() assert row.status == "error" and "No off-site destination" in row.error def test_a_long_media_run_is_judged_stalled_by_its_heartbeat(db_sync): from backend.app.tasks.maintenance import recover_stalled_backup_runs long_ago = datetime.now(UTC) - timedelta(hours=20) alive = BackupRun(kind="media", status="running", triggered_by="nightly", started_at=long_ago, last_progress_at=datetime.now(UTC), manifest={}) silent = BackupRun(kind="media", status="running", triggered_by="nightly", started_at=long_ago, last_progress_at=long_ago, manifest={}) db_sync.add_all([alive, silent]) db_sync.commit() assert recover_stalled_backup_runs() == 1 db_sync.refresh(alive) db_sync.refresh(silent) assert (alive.status, silent.status) == ("running", "error") # ── reconcile ─────────────────────────────────────────────────────────────── def test_reconcile_resends_what_the_bucket_lost_and_adopts_what_it_has( db_sync, tmp_path, monkeypatch, ): _image(db_sync, tmp_path, "a") # row, but the bucket lost it _image(db_sync, tmp_path, "b") # in the bucket, no row _image(db_sync, tmp_path, "c") # neither key_a = backup_media.blob_key(_sha("a"), "x.jpg") key_b = backup_media.blob_key(_sha("b"), "x.jpg") db_sync.add(BackupBlob(target=TID, sha256=_sha("a"), key=key_a, size_bytes=3)) db_sync.commit() pages = iter([ ListPage([StoredObject("media/zz/other", 1, "")], "next"), ListPage([StoredObject(key_b, 3, "")], None), ]) monkeypatch.setattr(backup_media, "list_objects", lambda *a, **kw: next(pages)) monkeypatch.setattr(backup_media.time, "sleep", lambda s: None) assert backup_media.reconcile(db_sync, TARGET) == { "listed": 2, "dropped": 1, "adopted": 1, } assert set(db_sync.execute(select(BackupBlob.sha256)).scalars()) == {_sha("b")}