Files
FabledCurator/tests/test_api_system_backup.py
T
bvandeusenandClaude Opus 5.5 375e7e2eda
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 1s
CI and images / extension-test (push) Successful in 17s
CI and images / frontend-build (push) Successful in 19s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Failing after 2m26s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped
feat: restore a new install from the off-site bucket, database first, then every file
Settings → Maintenance → Backups gains "Restore from the bucket": pick one of
the bucket's dated dumps, type its confirmation, and FabledCurator saves the
current database to the bucket, replaces it with the dump, migrates it to this
version, then fetches every image and attachment back to the path its row
records. Each file is checked against its sha256 on arrival and moved into
place only then; a file already on disk is left alone, so "Fetch missing
files" repairs a library that lost files without touching the database.

- services/backup_restore.py: list/find/fetch dumps, migrate, keep this
  install's destination (the dump's sealed secret can't be opened here),
  library_files work list, restore_file (one fetch per sha, copies for
  duplicate paths, refuses paths outside /images).
- restore_offsite_task: the database phase, then 45-minute file chunks with a
  heartbeat; a failed load leaves an empty migrated database and says why.
  The dump's own run row is marked ok, other runs then going "Interrupted".
- backup_s3.get_file: sha/size check on arrival, ObjectMissing for a 404,
  download pacing.
- backup_service: every dump leaves the credential table's rows out;
  restore_db reports pg_restore's last line, stops at the first error, ends
  other connections and waits at most 60 s for locks.
- Backups and orphan repair wait while a restore runs; the stall sweep judges
  restores by heartbeat; restore runs are pruned to 20.
- CLI: python -m backend.app.scripts.restore list|database|files.
- README: backup and restore section; logins are in no backup.
- Tests: a round trip through an in-process S3 bucket (tests/fake_s3.py) with
  real pg_dump/pg_restore, plus file, run, API and get_file cases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 14:34:47 -05:00

587 lines
20 KiB
Python

"""FC-3h: /api/system/backup/* endpoint integration tests."""
from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from sqlalchemy import select
from backend.app.models import BackupRun
pytestmark = pytest.mark.integration
@pytest_asyncio.fixture
async def _seed_runs(db):
"""Insert 4 BackupRun rows for paging/filter tests."""
now = datetime.now(UTC)
for i in range(4):
db.add(BackupRun(
kind="db" if i % 2 == 0 else "images",
status="ok",
tag=None,
triggered_by="manual",
started_at=now - timedelta(seconds=10 - i),
finished_at=now - timedelta(seconds=9 - i),
sql_path="/tmp/fake.sql" if i % 2 == 0 else None,
tar_path=None if i % 2 == 0 else "/tmp/fake.tar.zst",
size_bytes=100,
manifest={},
))
await db.commit()
# --- POST /db, /images ----------------------------------------------
@pytest.mark.asyncio
async def test_post_db_dispatches_and_returns_202(client, monkeypatch):
dispatched = []
monkeypatch.setattr(
"backend.app.tasks.backup.backup_db_task.delay",
lambda **kw: dispatched.append(kw),
)
resp = await client.post("/api/system/backup/db", json={})
assert resp.status_code == 202
body = await resp.get_json()
assert body["status"] == "dispatched"
assert dispatched == [{"tag": None, "triggered_by": "manual"}]
@pytest.mark.asyncio
async def test_post_db_persists_tag(client, monkeypatch):
dispatched = []
monkeypatch.setattr(
"backend.app.tasks.backup.backup_db_task.delay",
lambda **kw: dispatched.append(kw),
)
resp = await client.post(
"/api/system/backup/db", json={"tag": "pre-cutover"},
)
assert resp.status_code == 202
assert dispatched[0]["tag"] == "pre-cutover"
@pytest.mark.asyncio
async def test_post_db_rejects_tag_too_long(client):
resp = await client.post(
"/api/system/backup/db", json={"tag": "x" * 65},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "invalid_tag"
@pytest.mark.asyncio
async def test_post_images_dispatches(client, monkeypatch):
dispatched = []
monkeypatch.setattr(
"backend.app.tasks.backup.backup_images_task.delay",
lambda **kw: dispatched.append(kw),
)
resp = await client.post("/api/system/backup/images", json={})
assert resp.status_code == 202
# --- GET /runs -------------------------------------------------------
@pytest.mark.asyncio
async def test_runs_paginated_descending(client, _seed_runs):
resp = await client.get("/api/system/backup/runs?limit=2")
body = await resp.get_json()
assert len(body["runs"]) == 2
ids = [r["id"] for r in body["runs"]]
assert ids == sorted(ids, reverse=True)
assert body["next_cursor"] is not None
@pytest.mark.asyncio
async def test_runs_filter_by_kind_db(client, _seed_runs):
resp = await client.get("/api/system/backup/runs?kind=db")
body = await resp.get_json()
assert all(r["kind"] == "db" for r in body["runs"])
@pytest.mark.asyncio
async def test_runs_filter_by_kind_images(client, _seed_runs):
resp = await client.get("/api/system/backup/runs?kind=images")
body = await resp.get_json()
assert all(r["kind"] == "images" for r in body["runs"])
@pytest.mark.asyncio
async def test_runs_rejects_invalid_kind(client):
resp = await client.get("/api/system/backup/runs?kind=bogus")
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_runs_rejects_invalid_limit(client):
resp = await client.get("/api/system/backup/runs?limit=not-int")
assert resp.status_code == 400
# --- GET /runs/<id> --------------------------------------------------
@pytest.mark.asyncio
async def test_get_run_returns_row(client, _seed_runs):
list_resp = await client.get("/api/system/backup/runs?limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
resp = await client.get(f"/api/system/backup/runs/{rid}")
assert resp.status_code == 200
body = await resp.get_json()
assert body["id"] == rid
@pytest.mark.asyncio
async def test_get_run_404(client):
resp = await client.get("/api/system/backup/runs/999999")
assert resp.status_code == 404
# --- PATCH /runs/<id> ------------------------------------------------
@pytest.mark.asyncio
async def test_patch_tag_sets_value(client, _seed_runs, db_sync):
list_resp = await client.get("/api/system/backup/runs?limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
resp = await client.patch(
f"/api/system/backup/runs/{rid}",
json={"tag": "monthly"},
)
assert resp.status_code == 200
tag = db_sync.execute(
select(BackupRun.tag).where(BackupRun.id == rid)
).scalar_one()
assert tag == "monthly"
@pytest.mark.asyncio
async def test_patch_tag_null_clears(client, _seed_runs, db_sync):
list_resp = await client.get("/api/system/backup/runs?limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": "x"})
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": None})
tag = db_sync.execute(
select(BackupRun.tag).where(BackupRun.id == rid)
).scalar_one()
assert tag is None
# --- POST /runs/<id>/restore ----------------------------------------
@pytest.mark.asyncio
async def test_restore_wrong_confirm_400(client, _seed_runs):
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
resp = await client.post(
f"/api/system/backup/runs/{rid}/restore",
json={"confirm": "wrong"},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "confirm_mismatch"
assert body["expected"] == f"restore-db-{rid}"
@pytest.mark.asyncio
async def test_restore_correct_confirm_dispatches(client, _seed_runs, monkeypatch):
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
dispatched = []
monkeypatch.setattr(
"backend.app.tasks.backup.restore_db_task.delay",
lambda **kw: dispatched.append(kw),
)
resp = await client.post(
f"/api/system/backup/runs/{rid}/restore",
json={"confirm": f"restore-db-{rid}"},
)
assert resp.status_code == 202
assert dispatched == [{"source_backup_run_id": rid}]
@pytest.mark.asyncio
async def test_restore_rejects_non_ok_status(client, db):
# Seed an error-status row directly.
db.add(BackupRun(
kind="db", status="error", tag=None, triggered_by="manual",
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
sql_path="/tmp/fake.sql", manifest={},
))
await db.commit()
rid_q = await client.get("/api/system/backup/runs?kind=db&limit=1")
rid = (await rid_q.get_json())["runs"][0]["id"]
resp = await client.post(
f"/api/system/backup/runs/{rid}/restore",
json={"confirm": f"restore-db-{rid}"},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "not_restorable"
@pytest.mark.asyncio
async def test_an_off_site_dump_is_not_restored_from_this_screen(client, db):
db.add(BackupRun(
kind="db", status="ok", tag=None, triggered_by="nightly",
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
manifest={"offsite": {"endpoint": "http://archives.example:8081",
"bucket": "curator", "key": "database/x.dump"}},
))
await db.commit()
run = (await (await client.get("/api/system/backup/runs?kind=db&limit=1")).get_json())[
"runs"][0]
assert run["offsite"]["bucket"] == "curator"
resp = await client.post(
f"/api/system/backup/runs/{run['id']}/restore",
json={"confirm": f"restore-db-{run['id']}"},
)
assert resp.status_code == 400
assert (await resp.get_json())["error"] == "not_restorable"
# --- DELETE /runs/<id> ----------------------------------------------
@pytest.mark.asyncio
async def test_delete_wrong_confirm_400(client, _seed_runs):
list_resp = await client.get("/api/system/backup/runs?limit=1")
rid = (await list_resp.get_json())["runs"][0]["id"]
resp = await client.delete(
f"/api/system/backup/runs/{rid}",
json={"confirm": "wrong"},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_delete_correct_confirm_204(client, _seed_runs, db_sync):
list_resp = await client.get("/api/system/backup/runs?limit=1")
row = (await list_resp.get_json())["runs"][0]
rid = row["id"]
kind = row["kind"]
resp = await client.delete(
f"/api/system/backup/runs/{rid}",
json={"confirm": f"delete-{kind}-{rid}"},
)
assert resp.status_code == 204
surviving = db_sync.execute(
select(BackupRun.id).where(BackupRun.id == rid)
).scalar_one_or_none()
assert surviving is None
# --- /settings ------------------------------------------------------
@pytest.mark.asyncio
async def test_get_settings_returns_defaults(client):
resp = await client.get("/api/system/backup/settings")
assert resp.status_code == 200
body = await resp.get_json()
assert body["backup_db_nightly_enabled"] is False
assert body["backup_db_nightly_hour_utc"] == 3
assert body["backup_db_keep_last_n"] == 14
assert body["backup_images_keep_last_n"] == 3
@pytest.mark.asyncio
async def test_patch_settings_updates_values(client):
resp = await client.patch(
"/api/system/backup/settings",
json={
"backup_db_nightly_enabled": True,
"backup_db_nightly_hour_utc": 5,
"backup_db_keep_last_n": 30,
},
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["backup_db_nightly_enabled"] is True
assert body["backup_db_nightly_hour_utc"] == 5
assert body["backup_db_keep_last_n"] == 30
@pytest.mark.asyncio
async def test_patch_settings_rejects_invalid_bool(client):
resp = await client.patch(
"/api/system/backup/settings",
json={"backup_db_nightly_enabled": "yes"},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_patch_settings_rejects_hour_out_of_range(client):
resp = await client.patch(
"/api/system/backup/settings",
json={"backup_db_nightly_hour_utc": 24},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_patch_settings_rejects_keep_n_below_min(client):
resp = await client.patch(
"/api/system/backup/settings",
json={"backup_db_keep_last_n": 0},
)
assert resp.status_code == 400
# --- /destination ---------------------------------------------------
@pytest.fixture
def _sealing_key(tmp_path, monkeypatch):
from backend.app.services import backup_destination
monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
_DEST = {
"endpoint": "http://archives.example:8081/",
"bucket": "curator",
"access_key": "AKEXAMPLE",
"region": "",
"secret_key": "s3cr3t-example",
}
@pytest.mark.asyncio
async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
from backend.app.models import ImportSettings
resp = await client.patch("/api/system/backup/destination", json=_DEST)
assert resp.status_code == 200
body = await resp.get_json()
assert body == {
"endpoint": "http://archives.example:8081", "bucket": "curator",
"access_key": "AKEXAMPLE", "region": "us-east-1",
"secret_set": True, "configured": True, "upload_limit_mbps": 200,
}
assert "s3cr3t-example" not in str(await (await client.get(
"/api/system/backup/destination")).get_json())
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
db_sync.refresh(row)
assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
from backend.app.services import backup_destination
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
@pytest.mark.asyncio
async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
from backend.app.models import ImportSettings
from backend.app.services import backup_destination
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.patch(
"/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
)
assert (await resp.get_json())["bucket"] == "other"
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
db_sync.refresh(row)
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
@pytest.mark.asyncio
@pytest.mark.parametrize("patch, word", [
({"endpoint": "archives.example:8081"}, "http://"),
({"endpoint": "ftp://archives.example"}, "http://"),
({"bucket": "curator/sub"}, "bucket"),
({"access_key": 5}, "access_key"),
])
async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
assert resp.status_code == 400
assert word in (await resp.get_json())["detail"]
@pytest.mark.asyncio
async def test_destination_remove_forgets_everything(client, _sealing_key):
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.delete("/api/system/backup/destination")
assert await resp.get_json() == {
"endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
"secret_set": False, "configured": False, "upload_limit_mbps": 200,
}
@pytest.mark.asyncio
async def test_destination_test_uses_the_form_and_the_saved_secret(
client, _sealing_key, monkeypatch,
):
from backend.app.services import backup_destination
seen = []
def fake_check(target):
seen.append(target)
return {"ok": True, "message": "Reached it."}
monkeypatch.setattr(backup_destination, "check_target", fake_check)
await client.patch("/api/system/backup/destination", json=_DEST)
resp = await client.post(
"/api/system/backup/destination/test",
json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
)
assert await resp.get_json() == {"ok": True, "message": "Reached it."}
assert seen[0].bucket == "typed-not-saved"
assert seen[0].secret_key == "s3cr3t-example"
assert seen[0].endpoint == "http://archives.example:8081"
@pytest.mark.asyncio
async def test_destination_test_names_what_is_missing(client, _sealing_key):
await client.delete("/api/system/backup/destination")
resp = await client.post(
"/api/system/backup/destination/test",
json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
)
body = await resp.get_json()
assert body["ok"] is False
assert body["message"] == "Not tested: fill in the access key, secret key."
@pytest.mark.asyncio
async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
client, _sealing_key, monkeypatch,
):
from backend.app.services import backup_destination
from backend.app.services.backup_s3 import S3Error
def refuse(target, **kw):
raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
monkeypatch.setattr(backup_destination, "list_objects", refuse)
resp = await client.post("/api/system/backup/destination/test", json=_DEST)
assert resp.status_code == 200
assert await resp.get_json() == {
"ok": False,
"message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
}
@pytest.mark.asyncio
async def test_the_upload_limit_saves_and_a_missing_one_keeps_it(client, _sealing_key):
resp = await client.patch(
"/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": 50},
)
assert (await resp.get_json())["upload_limit_mbps"] == 50
resp = await client.patch("/api/system/backup/destination", json=_DEST)
assert (await resp.get_json())["upload_limit_mbps"] == 50
@pytest.mark.asyncio
@pytest.mark.parametrize("limit", [-1, 2.5, "fast", True])
async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _sealing_key, limit):
resp = await client.patch(
"/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": limit},
)
assert resp.status_code == 400
assert "upload_limit_mbps" in (await resp.get_json())["detail"]
# ── restoring from the off-site bucket ──────────────────────────────────────
@pytest.fixture
def _saved_bucket(monkeypatch):
from backend.app.services import backup_destination
from backend.app.services.backup_s3 import S3Target
target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
monkeypatch.setattr(backup_destination, "saved_target", lambda settings: target)
dispatched = []
monkeypatch.setattr("backend.app.tasks.backup.restore_offsite_task.delay",
lambda **kw: dispatched.append(kw))
return dispatched
_DUMP = "database/fabledcurator-20261107T134258Z.dump"
@pytest.mark.asyncio
async def test_the_bucket_dumps_are_listed_with_their_confirmation(
client, _saved_bucket, monkeypatch,
):
from backend.app.services import backup_restore
monkeypatch.setattr(backup_restore, "list_dumps", lambda target: [
backup_restore.Dump(_DUMP, "20261107T134258Z", 3222868924),
])
resp = await client.get("/api/system/backup/offsite/dumps")
assert resp.status_code == 200
assert (await resp.get_json())["dumps"] == [{
"key": _DUMP, "ts": "20261107T134258Z", "size_bytes": 3222868924,
"confirm": "restore-offsite-20261107T134258Z",
}]
@pytest.mark.asyncio
async def test_a_restore_needs_the_dump_typed_back(client, _saved_bucket):
resp = await client.post("/api/system/backup/offsite/restore",
json={"key": _DUMP, "confirm": "yes"})
assert resp.status_code == 400
assert (await resp.get_json())["expected"] == "restore-offsite-20261107T134258Z"
assert _saved_bucket == []
resp = await client.post("/api/system/backup/offsite/restore", json={
"key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
assert resp.status_code == 202
assert _saved_bucket == [{"dump_key": _DUMP}]
@pytest.mark.asyncio
async def test_a_restore_waits_while_a_backup_runs(client, _saved_bucket, db_sync):
db_sync.add(BackupRun(kind="db", status="running", triggered_by="nightly",
started_at=datetime.now(UTC), manifest={}))
db_sync.commit()
resp = await client.post("/api/system/backup/offsite/restore", json={
"key": _DUMP, "confirm": "restore-offsite-20261107T134258Z"})
assert resp.status_code == 409
assert "db backup is running" in (await resp.get_json())["detail"]
resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
assert resp.status_code == 409
assert _saved_bucket == []
@pytest.mark.asyncio
async def test_fetching_missing_files_leaves_the_database(client, _saved_bucket):
resp = await client.post("/api/system/backup/offsite/fetch-files", json={})
assert resp.status_code == 202
assert _saved_bucket == [{"files_only": True}]
@pytest.mark.asyncio
async def test_restoring_needs_a_destination(client):
resp = await client.get("/api/system/backup/offsite/dumps")
assert resp.status_code == 400
assert (await resp.get_json())["error"] == "no_destination"
@pytest.mark.asyncio
async def test_restore_runs_are_listed_as_their_own_kind(client, db_sync):
db_sync.add(BackupRun(kind="restore", status="restoring", triggered_by="restore",
started_at=datetime.now(UTC), manifest={"phase": "files"}))
db_sync.commit()
resp = await client.get("/api/system/backup/runs?kind=restore")
runs = (await resp.get_json())["runs"]
assert [r["status"] for r in runs] == ["restoring"]