bvandeusenandClaude Opus 5.5
772482d794
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 1s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 41s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m50s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 2s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The nightly database backup (and "Run DB backup now") now sends pg_dump's output straight to the off-site bucket as a multipart upload, with no copy staged on this server. The object is named database/fabledcurator-<UTC time>.dump and appears only once pg_dump has exited cleanly and every part has landed; a failed dump aborts the upload and records pg_dump's last line. Without a destination, dumps are written to /images/_backups as before. - backup_s3: Upload (start / add part / complete / abort), with a busy or silent bucket rested and asked again (15 s, 60 s, Retry-After obeyed), and a Pacer that holds uploads to the new bandwidth limit. - Settings: "Upload limit (Mbit/s)" on the destination card, 200 by default, 0 for none (migration 0120). - pg_dump runs under nice (and ionice where installed), locally too. - Backups card says where dumps go; the runs table gains "Saved to" and shows a failed run's error. Off-site rows have no restore/tag/delete here: restoring from the bucket is a later step (#6880). Task #6878, milestone 586. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
499 lines
16 KiB
Python
499 lines
16 KiB
Python
"""FC-3h: /api/system/backup/* endpoint integration tests."""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import BackupRun
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def _seed_runs(db):
|
|
"""Insert 4 BackupRun rows for paging/filter tests."""
|
|
now = datetime.now(UTC)
|
|
for i in range(4):
|
|
db.add(BackupRun(
|
|
kind="db" if i % 2 == 0 else "images",
|
|
status="ok",
|
|
tag=None,
|
|
triggered_by="manual",
|
|
started_at=now - timedelta(seconds=10 - i),
|
|
finished_at=now - timedelta(seconds=9 - i),
|
|
sql_path="/tmp/fake.sql" if i % 2 == 0 else None,
|
|
tar_path=None if i % 2 == 0 else "/tmp/fake.tar.zst",
|
|
size_bytes=100,
|
|
manifest={},
|
|
))
|
|
await db.commit()
|
|
|
|
|
|
# --- POST /db, /images ----------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_db_dispatches_and_returns_202(client, monkeypatch):
|
|
dispatched = []
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.backup.backup_db_task.delay",
|
|
lambda **kw: dispatched.append(kw),
|
|
)
|
|
resp = await client.post("/api/system/backup/db", json={})
|
|
assert resp.status_code == 202
|
|
body = await resp.get_json()
|
|
assert body["status"] == "dispatched"
|
|
assert dispatched == [{"tag": None, "triggered_by": "manual"}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_db_persists_tag(client, monkeypatch):
|
|
dispatched = []
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.backup.backup_db_task.delay",
|
|
lambda **kw: dispatched.append(kw),
|
|
)
|
|
resp = await client.post(
|
|
"/api/system/backup/db", json={"tag": "pre-cutover"},
|
|
)
|
|
assert resp.status_code == 202
|
|
assert dispatched[0]["tag"] == "pre-cutover"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_db_rejects_tag_too_long(client):
|
|
resp = await client.post(
|
|
"/api/system/backup/db", json={"tag": "x" * 65},
|
|
)
|
|
assert resp.status_code == 400
|
|
body = await resp.get_json()
|
|
assert body["error"] == "invalid_tag"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_images_dispatches(client, monkeypatch):
|
|
dispatched = []
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.backup.backup_images_task.delay",
|
|
lambda **kw: dispatched.append(kw),
|
|
)
|
|
resp = await client.post("/api/system/backup/images", json={})
|
|
assert resp.status_code == 202
|
|
|
|
|
|
# --- GET /runs -------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runs_paginated_descending(client, _seed_runs):
|
|
resp = await client.get("/api/system/backup/runs?limit=2")
|
|
body = await resp.get_json()
|
|
assert len(body["runs"]) == 2
|
|
ids = [r["id"] for r in body["runs"]]
|
|
assert ids == sorted(ids, reverse=True)
|
|
assert body["next_cursor"] is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runs_filter_by_kind_db(client, _seed_runs):
|
|
resp = await client.get("/api/system/backup/runs?kind=db")
|
|
body = await resp.get_json()
|
|
assert all(r["kind"] == "db" for r in body["runs"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runs_filter_by_kind_images(client, _seed_runs):
|
|
resp = await client.get("/api/system/backup/runs?kind=images")
|
|
body = await resp.get_json()
|
|
assert all(r["kind"] == "images" for r in body["runs"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runs_rejects_invalid_kind(client):
|
|
resp = await client.get("/api/system/backup/runs?kind=bogus")
|
|
assert resp.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runs_rejects_invalid_limit(client):
|
|
resp = await client.get("/api/system/backup/runs?limit=not-int")
|
|
assert resp.status_code == 400
|
|
|
|
|
|
# --- GET /runs/<id> --------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_run_returns_row(client, _seed_runs):
|
|
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
resp = await client.get(f"/api/system/backup/runs/{rid}")
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body["id"] == rid
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_run_404(client):
|
|
resp = await client.get("/api/system/backup/runs/999999")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# --- PATCH /runs/<id> ------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_tag_sets_value(client, _seed_runs, db_sync):
|
|
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
|
|
resp = await client.patch(
|
|
f"/api/system/backup/runs/{rid}",
|
|
json={"tag": "monthly"},
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
tag = db_sync.execute(
|
|
select(BackupRun.tag).where(BackupRun.id == rid)
|
|
).scalar_one()
|
|
assert tag == "monthly"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_tag_null_clears(client, _seed_runs, db_sync):
|
|
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
|
|
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": "x"})
|
|
await client.patch(f"/api/system/backup/runs/{rid}", json={"tag": None})
|
|
|
|
tag = db_sync.execute(
|
|
select(BackupRun.tag).where(BackupRun.id == rid)
|
|
).scalar_one()
|
|
assert tag is None
|
|
|
|
|
|
# --- POST /runs/<id>/restore ----------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restore_wrong_confirm_400(client, _seed_runs):
|
|
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
|
|
resp = await client.post(
|
|
f"/api/system/backup/runs/{rid}/restore",
|
|
json={"confirm": "wrong"},
|
|
)
|
|
assert resp.status_code == 400
|
|
body = await resp.get_json()
|
|
assert body["error"] == "confirm_mismatch"
|
|
assert body["expected"] == f"restore-db-{rid}"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restore_correct_confirm_dispatches(client, _seed_runs, monkeypatch):
|
|
list_resp = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
|
|
dispatched = []
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.backup.restore_db_task.delay",
|
|
lambda **kw: dispatched.append(kw),
|
|
)
|
|
|
|
resp = await client.post(
|
|
f"/api/system/backup/runs/{rid}/restore",
|
|
json={"confirm": f"restore-db-{rid}"},
|
|
)
|
|
assert resp.status_code == 202
|
|
assert dispatched == [{"source_backup_run_id": rid}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restore_rejects_non_ok_status(client, db):
|
|
# Seed an error-status row directly.
|
|
db.add(BackupRun(
|
|
kind="db", status="error", tag=None, triggered_by="manual",
|
|
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
|
|
sql_path="/tmp/fake.sql", manifest={},
|
|
))
|
|
await db.commit()
|
|
rid_q = await client.get("/api/system/backup/runs?kind=db&limit=1")
|
|
rid = (await rid_q.get_json())["runs"][0]["id"]
|
|
|
|
resp = await client.post(
|
|
f"/api/system/backup/runs/{rid}/restore",
|
|
json={"confirm": f"restore-db-{rid}"},
|
|
)
|
|
assert resp.status_code == 400
|
|
body = await resp.get_json()
|
|
assert body["error"] == "not_restorable"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_an_off_site_dump_is_not_restored_from_this_screen(client, db):
|
|
db.add(BackupRun(
|
|
kind="db", status="ok", tag=None, triggered_by="nightly",
|
|
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
|
|
manifest={"offsite": {"endpoint": "http://archives.example:8081",
|
|
"bucket": "curator", "key": "database/x.dump"}},
|
|
))
|
|
await db.commit()
|
|
run = (await (await client.get("/api/system/backup/runs?kind=db&limit=1")).get_json())[
|
|
"runs"][0]
|
|
assert run["offsite"]["bucket"] == "curator"
|
|
|
|
resp = await client.post(
|
|
f"/api/system/backup/runs/{run['id']}/restore",
|
|
json={"confirm": f"restore-db-{run['id']}"},
|
|
)
|
|
assert resp.status_code == 400
|
|
assert (await resp.get_json())["error"] == "not_restorable"
|
|
|
|
|
|
# --- DELETE /runs/<id> ----------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_wrong_confirm_400(client, _seed_runs):
|
|
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
|
rid = (await list_resp.get_json())["runs"][0]["id"]
|
|
|
|
resp = await client.delete(
|
|
f"/api/system/backup/runs/{rid}",
|
|
json={"confirm": "wrong"},
|
|
)
|
|
assert resp.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_correct_confirm_204(client, _seed_runs, db_sync):
|
|
list_resp = await client.get("/api/system/backup/runs?limit=1")
|
|
row = (await list_resp.get_json())["runs"][0]
|
|
rid = row["id"]
|
|
kind = row["kind"]
|
|
|
|
resp = await client.delete(
|
|
f"/api/system/backup/runs/{rid}",
|
|
json={"confirm": f"delete-{kind}-{rid}"},
|
|
)
|
|
assert resp.status_code == 204
|
|
|
|
surviving = db_sync.execute(
|
|
select(BackupRun.id).where(BackupRun.id == rid)
|
|
).scalar_one_or_none()
|
|
assert surviving is None
|
|
|
|
|
|
# --- /settings ------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_settings_returns_defaults(client):
|
|
resp = await client.get("/api/system/backup/settings")
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body["backup_db_nightly_enabled"] is False
|
|
assert body["backup_db_nightly_hour_utc"] == 3
|
|
assert body["backup_db_keep_last_n"] == 14
|
|
assert body["backup_images_keep_last_n"] == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_settings_updates_values(client):
|
|
resp = await client.patch(
|
|
"/api/system/backup/settings",
|
|
json={
|
|
"backup_db_nightly_enabled": True,
|
|
"backup_db_nightly_hour_utc": 5,
|
|
"backup_db_keep_last_n": 30,
|
|
},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body["backup_db_nightly_enabled"] is True
|
|
assert body["backup_db_nightly_hour_utc"] == 5
|
|
assert body["backup_db_keep_last_n"] == 30
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_settings_rejects_invalid_bool(client):
|
|
resp = await client.patch(
|
|
"/api/system/backup/settings",
|
|
json={"backup_db_nightly_enabled": "yes"},
|
|
)
|
|
assert resp.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_settings_rejects_hour_out_of_range(client):
|
|
resp = await client.patch(
|
|
"/api/system/backup/settings",
|
|
json={"backup_db_nightly_hour_utc": 24},
|
|
)
|
|
assert resp.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_patch_settings_rejects_keep_n_below_min(client):
|
|
resp = await client.patch(
|
|
"/api/system/backup/settings",
|
|
json={"backup_db_keep_last_n": 0},
|
|
)
|
|
assert resp.status_code == 400
|
|
|
|
|
|
# --- /destination ---------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def _sealing_key(tmp_path, monkeypatch):
|
|
from backend.app.services import backup_destination
|
|
monkeypatch.setattr(backup_destination, "KEY_PATH", tmp_path / "credential_key.b64")
|
|
|
|
|
|
_DEST = {
|
|
"endpoint": "http://archives.example:8081/",
|
|
"bucket": "curator",
|
|
"access_key": "AKEXAMPLE",
|
|
"region": "",
|
|
"secret_key": "s3cr3t-example",
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_saves_and_never_returns_the_secret(client, db_sync, _sealing_key):
|
|
from backend.app.models import ImportSettings
|
|
|
|
resp = await client.patch("/api/system/backup/destination", json=_DEST)
|
|
assert resp.status_code == 200
|
|
body = await resp.get_json()
|
|
assert body == {
|
|
"endpoint": "http://archives.example:8081", "bucket": "curator",
|
|
"access_key": "AKEXAMPLE", "region": "us-east-1",
|
|
"secret_set": True, "configured": True, "upload_limit_mbps": 200,
|
|
}
|
|
assert "s3cr3t-example" not in str(await (await client.get(
|
|
"/api/system/backup/destination")).get_json())
|
|
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
|
|
db_sync.refresh(row)
|
|
assert row.backup_s3_secret_sealed and "s3cr3t-example" not in row.backup_s3_secret_sealed
|
|
|
|
from backend.app.services import backup_destination
|
|
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_blank_secret_keeps_the_saved_one(client, db_sync, _sealing_key):
|
|
from backend.app.models import ImportSettings
|
|
from backend.app.services import backup_destination
|
|
|
|
await client.patch("/api/system/backup/destination", json=_DEST)
|
|
resp = await client.patch(
|
|
"/api/system/backup/destination", json={**_DEST, "bucket": "other", "secret_key": ""},
|
|
)
|
|
assert (await resp.get_json())["bucket"] == "other"
|
|
row = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one()
|
|
db_sync.refresh(row)
|
|
assert backup_destination.unseal(row.backup_s3_secret_sealed) == "s3cr3t-example"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("patch, word", [
|
|
({"endpoint": "archives.example:8081"}, "http://"),
|
|
({"endpoint": "ftp://archives.example"}, "http://"),
|
|
({"bucket": "curator/sub"}, "bucket"),
|
|
({"access_key": 5}, "access_key"),
|
|
])
|
|
async def test_destination_rejects_what_cant_be_an_s3_target(client, _sealing_key, patch, word):
|
|
resp = await client.patch("/api/system/backup/destination", json={**_DEST, **patch})
|
|
assert resp.status_code == 400
|
|
assert word in (await resp.get_json())["detail"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_remove_forgets_everything(client, _sealing_key):
|
|
await client.patch("/api/system/backup/destination", json=_DEST)
|
|
resp = await client.delete("/api/system/backup/destination")
|
|
assert await resp.get_json() == {
|
|
"endpoint": "", "bucket": "", "access_key": "", "region": "us-east-1",
|
|
"secret_set": False, "configured": False, "upload_limit_mbps": 200,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_test_uses_the_form_and_the_saved_secret(
|
|
client, _sealing_key, monkeypatch,
|
|
):
|
|
from backend.app.services import backup_destination
|
|
|
|
seen = []
|
|
|
|
def fake_check(target):
|
|
seen.append(target)
|
|
return {"ok": True, "message": "Reached it."}
|
|
|
|
monkeypatch.setattr(backup_destination, "check_target", fake_check)
|
|
await client.patch("/api/system/backup/destination", json=_DEST)
|
|
resp = await client.post(
|
|
"/api/system/backup/destination/test",
|
|
json={**_DEST, "bucket": "typed-not-saved", "secret_key": ""},
|
|
)
|
|
assert await resp.get_json() == {"ok": True, "message": "Reached it."}
|
|
assert seen[0].bucket == "typed-not-saved"
|
|
assert seen[0].secret_key == "s3cr3t-example"
|
|
assert seen[0].endpoint == "http://archives.example:8081"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_test_names_what_is_missing(client, _sealing_key):
|
|
await client.delete("/api/system/backup/destination")
|
|
resp = await client.post(
|
|
"/api/system/backup/destination/test",
|
|
json={"endpoint": "http://archives.example:8081", "bucket": "curator"},
|
|
)
|
|
body = await resp.get_json()
|
|
assert body["ok"] is False
|
|
assert body["message"] == "Not tested: fill in the access key, secret key."
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_destination_test_reports_an_unreachable_bucket_as_a_result(
|
|
client, _sealing_key, monkeypatch,
|
|
):
|
|
from backend.app.services import backup_destination
|
|
from backend.app.services.backup_s3 import S3Error
|
|
|
|
def refuse(target, **kw):
|
|
raise S3Error("Couldn't reach http://archives.example:8081: nothing is listening there.")
|
|
|
|
monkeypatch.setattr(backup_destination, "list_objects", refuse)
|
|
resp = await client.post("/api/system/backup/destination/test", json=_DEST)
|
|
assert resp.status_code == 200
|
|
assert await resp.get_json() == {
|
|
"ok": False,
|
|
"message": "Couldn't reach http://archives.example:8081: nothing is listening there.",
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_the_upload_limit_saves_and_a_missing_one_keeps_it(client, _sealing_key):
|
|
resp = await client.patch(
|
|
"/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": 50},
|
|
)
|
|
assert (await resp.get_json())["upload_limit_mbps"] == 50
|
|
resp = await client.patch("/api/system/backup/destination", json=_DEST)
|
|
assert (await resp.get_json())["upload_limit_mbps"] == 50
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("limit", [-1, 2.5, "fast", True])
|
|
async def test_an_upload_limit_that_isnt_a_whole_number_is_refused(client, _sealing_key, limit):
|
|
resp = await client.patch(
|
|
"/api/system/backup/destination", json={**_DEST, "upload_limit_mbps": limit},
|
|
)
|
|
assert resp.status_code == 400
|
|
assert "upload_limit_mbps" in (await resp.get_json())["detail"]
|