feat: with an off-site destination set up, database backups stream to it
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 1s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 41s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m50s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 2s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 1s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 41s
CI and images / integration (push) Successful in 2m25s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m50s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 2s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The nightly database backup (and "Run DB backup now") now sends pg_dump's output straight to the off-site bucket as a multipart upload, with no copy staged on this server. The object is named database/fabledcurator-<UTC time>.dump and appears only once pg_dump has exited cleanly and every part has landed; a failed dump aborts the upload and records pg_dump's last line. Without a destination, dumps are written to /images/_backups as before. - backup_s3: Upload (start / add part / complete / abort), with a busy or silent bucket rested and asked again (15 s, 60 s, Retry-After obeyed), and a Pacer that holds uploads to the new bandwidth limit. - Settings: "Upload limit (Mbit/s)" on the destination card, 200 by default, 0 for none (migration 0120). - pg_dump runs under nice (and ionice where installed), locally too. - Backups card says where dumps go; the runs table gains "Saved to" and shows a failed run's error. Off-site rows have no restore/tag/delete here: restoring from the bucket is a later step (#6880). Task #6878, milestone 586. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,7 +33,7 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path):
|
||||
stderr = b""
|
||||
|
||||
def _sentinel(cmd):
|
||||
if cmd[0] == "pg_dump":
|
||||
if "pg_dump" in cmd:
|
||||
i = cmd.index("-f")
|
||||
Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
|
||||
elif cmd[0] == "tar" and "-cf" in cmd:
|
||||
@@ -128,6 +128,63 @@ async def test_backup_db_task_persists_tag(db_sync):
|
||||
assert tag == "pre-cutover"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def archives(monkeypatch):
|
||||
"""An off-site destination is set up; the dump itself is faked."""
|
||||
from backend.app.services.backup_s3 import S3Target
|
||||
target = S3Target("http://archives.example:8081", "curator", "AK", "SK")
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.backup_destination.saved_target", lambda settings: target,
|
||||
)
|
||||
sent = []
|
||||
|
||||
def _offsite(*, db_url, target, limit_mbps):
|
||||
sent.append({"target": target, "limit_mbps": limit_mbps})
|
||||
return {
|
||||
"kind": "db", "ts": "20261107T030000Z",
|
||||
"key": "database/fabledcurator-20261107T030000Z.dump",
|
||||
"bucket": target.bucket, "endpoint": target.endpoint,
|
||||
"parts": 3, "size_bytes": 1234,
|
||||
}
|
||||
monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _offsite)
|
||||
return sent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_with_a_destination_the_dump_goes_off_site_and_nothing_is_written_here(
|
||||
db_sync, archives, tmp_path,
|
||||
):
|
||||
from backend.app.tasks.backup import backup_db_task
|
||||
run_id = backup_db_task.delay(triggered_by="nightly").get()["backup_run_id"]
|
||||
row = db_sync.get(BackupRun, run_id)
|
||||
assert row.status == "ok"
|
||||
assert row.sql_path is None
|
||||
assert row.size_bytes == 1234
|
||||
assert row.manifest["offsite"] == {
|
||||
"endpoint": "http://archives.example:8081", "bucket": "curator",
|
||||
"key": "database/fabledcurator-20261107T030000Z.dump", "parts": 3,
|
||||
}
|
||||
assert archives[0]["limit_mbps"] == 200 # the setting's default
|
||||
assert not (tmp_path / "_backups").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_off_site_dump_that_fails_is_recorded_with_why(db_sync, monkeypatch, archives):
|
||||
from backend.app.services.backup_service import DumpFailed
|
||||
from backend.app.tasks.backup import backup_db_task
|
||||
|
||||
def _fails(**kwargs):
|
||||
raise DumpFailed("pg_dump: error: connection lost")
|
||||
monkeypatch.setattr("backend.app.services.backup_service.backup_db_offsite", _fails)
|
||||
with pytest.raises(DumpFailed):
|
||||
backup_db_task.delay().get()
|
||||
row = db_sync.execute(
|
||||
select(BackupRun).where(BackupRun.kind == "db").order_by(BackupRun.id.desc()).limit(1)
|
||||
).scalar_one()
|
||||
assert row.status == "error"
|
||||
assert row.error == "DumpFailed: pg_dump: error: connection lost"
|
||||
|
||||
|
||||
# --- backup_images_task ---------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user