fix: a media backup or restore cut off by a restart carries on where it stopped
CI and images / lint (push) Failing after 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 20s
CI and images / backend-lint-and-test (push) Successful in 46s
CI and images / integration (push) Failing after 3m8s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped
CI and images / lint (push) Failing after 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 20s
CI and images / backend-lint-and-test (push) Successful in 46s
CI and images / integration (push) Failing after 3m8s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped
A media backup and a restore's file phase run as chains of 45-minute chunks,
each queuing the next when it ends. A restart killed the chunk in flight, so
nothing queued the next one; the run's row stayed "running", "Back up now"
skipped it as already running, and only the 7-hour stall sweep ended it.
- When the long-maintenance lane starts, it queues the next chunk of every
media run still running and every restore fetching its files.
- Each chunk writes its token to the run and hands it to the chunk it queues.
A chunk queued by any other chunk is dropped (a killed chunk's message
redelivered by acks_late, say), so a resume never doubles the chain; a chunk
taken over stops at its next file.
- The integration job installs postgresql-client when the CI image lacks it:
the backup round trip in 375e7e2 runs the real pg_dump and pg_restore.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -197,10 +197,72 @@ def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, mon
|
||||
|
||||
result = tasks.backup_media_task(triggered_by="manual")
|
||||
assert result["continues"] is True
|
||||
assert run_env == [{"run_id": result["backup_run_id"]}]
|
||||
row = db_sync.get(BackupRun, result["backup_run_id"])
|
||||
db_sync.refresh(row)
|
||||
assert row.status == "running" and row.last_progress_at is not None
|
||||
# The next chunk carries the token of the chunk that queued it.
|
||||
assert run_env == [{"run_id": row.id, "after": row.manifest["chunk"]}]
|
||||
|
||||
|
||||
def _going(db_sync, chunk):
|
||||
row = BackupRun(kind="media", status="running", triggered_by="manual",
|
||||
started_at=datetime.now(UTC), size_bytes=0,
|
||||
manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
|
||||
"chunk": chunk})
|
||||
db_sync.add(row)
|
||||
db_sync.commit()
|
||||
return row
|
||||
|
||||
|
||||
def test_only_the_chunk_queued_by_the_one_carrying_the_run_goes_on(
|
||||
db_sync, tmp_path, run_env, monkeypatch,
|
||||
):
|
||||
from backend.app.tasks.backup import backup_media_task
|
||||
_image(db_sync, tmp_path, "a")
|
||||
row = _going(db_sync, "live")
|
||||
monkeypatch.setattr(backup_media, "send", _send_all_but(None))
|
||||
|
||||
# A killed chunk's own message, delivered again, is dropped.
|
||||
stale = backup_media_task(run_id=row.id, after="earlier")
|
||||
assert "another chunk" in stale["skipped"]
|
||||
assert backup_media_task(run_id=row.id, after="live")["continues"] is False
|
||||
db_sync.refresh(row)
|
||||
assert row.status == "ok" and row.manifest["sent"] == 1
|
||||
|
||||
|
||||
def test_a_resume_takes_the_run_and_the_chunk_it_replaced_stops(
|
||||
db_sync, tmp_path, run_env, monkeypatch,
|
||||
):
|
||||
from backend.app.tasks import backup as tasks
|
||||
for c in "ab":
|
||||
_image(db_sync, tmp_path, c)
|
||||
row = _going(db_sync, "killed")
|
||||
|
||||
def send_then_lose_the_run(target, blob, images_root, pacer):
|
||||
# While this chunk sends its first file, a newer one takes the run.
|
||||
with tasks._sync_session_factory()() as s:
|
||||
other = s.get(BackupRun, row.id)
|
||||
other.manifest = {**other.manifest, "chunk": "newer"}
|
||||
s.commit()
|
||||
return backup_media.blob_key(blob.sha256, blob.path)
|
||||
monkeypatch.setattr(backup_media, "send", send_then_lose_the_run)
|
||||
|
||||
result = tasks.backup_media_task(run_id=row.id, resume=True)
|
||||
assert "took the run over" in result["skipped"]
|
||||
assert run_env == []
|
||||
db_sync.refresh(row)
|
||||
assert row.status == "running" and row.manifest["chunk"] == "newer"
|
||||
assert row.manifest["sent"] == 0 and row.manifest["cursor"] == ""
|
||||
|
||||
|
||||
def test_a_restart_resumes_every_media_run_still_going(db_sync, run_env):
|
||||
from backend.app.tasks.backup import resume_offsite_runs
|
||||
going = _going(db_sync, "killed")
|
||||
db_sync.add(BackupRun(kind="media", status="ok", triggered_by="manual",
|
||||
started_at=datetime.now(UTC), manifest={}))
|
||||
db_sync.commit()
|
||||
assert resume_offsite_runs()["media"] == [going.id]
|
||||
assert run_env == [{"run_id": going.id, "resume": True}]
|
||||
|
||||
|
||||
def test_a_second_run_waits_for_the_one_going(db_sync, run_env):
|
||||
|
||||
Reference in New Issue
Block a user