fix: a media backup or restore cut off by a restart carries on where it stopped
CI and images / lint (push) Failing after 1s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 20s
CI and images / backend-lint-and-test (push) Successful in 46s
CI and images / integration (push) Failing after 3m8s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped

A media backup and a restore's file phase run as chains of 45-minute chunks,
each queuing the next when it ends. A restart killed the chunk in flight, so
nothing queued the next one; the run's row stayed "running", "Back up now"
skipped it as already running, and only the 7-hour stall sweep ended it.

- When the long-maintenance lane starts, it queues the next chunk of every
  media run still running and every restore fetching its files.
- Each chunk writes its token to the run and hands it to the chunk it queues.
  A chunk queued by any other chunk is dropped (a killed chunk's message
  redelivered by acks_late, say), so a resume never doubles the chain; a chunk
  taken over stops at its next file.
- The integration job installs postgresql-client when the CI image lacks it:
  the backup round trip in 375e7e2 runs the real pg_dump and pg_restore.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-07 14:53:57 -05:00
co-authored by Claude Opus 5.5
parent 375e7e2eda
commit 1e63f70bf7
7 changed files with 281 additions and 45 deletions
+63 -1
View File
@@ -197,10 +197,72 @@ def test_a_run_out_of_time_queues_its_next_chunk(db_sync, tmp_path, run_env, mon
result = tasks.backup_media_task(triggered_by="manual")
assert result["continues"] is True
assert run_env == [{"run_id": result["backup_run_id"]}]
row = db_sync.get(BackupRun, result["backup_run_id"])
db_sync.refresh(row)
assert row.status == "running" and row.last_progress_at is not None
# The next chunk carries the token of the chunk that queued it.
assert run_env == [{"run_id": row.id, "after": row.manifest["chunk"]}]
def _going(db_sync, chunk):
row = BackupRun(kind="media", status="running", triggered_by="manual",
started_at=datetime.now(UTC), size_bytes=0,
manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
"chunk": chunk})
db_sync.add(row)
db_sync.commit()
return row
def test_only_the_chunk_queued_by_the_one_carrying_the_run_goes_on(
db_sync, tmp_path, run_env, monkeypatch,
):
from backend.app.tasks.backup import backup_media_task
_image(db_sync, tmp_path, "a")
row = _going(db_sync, "live")
monkeypatch.setattr(backup_media, "send", _send_all_but(None))
# A killed chunk's own message, delivered again, is dropped.
stale = backup_media_task(run_id=row.id, after="earlier")
assert "another chunk" in stale["skipped"]
assert backup_media_task(run_id=row.id, after="live")["continues"] is False
db_sync.refresh(row)
assert row.status == "ok" and row.manifest["sent"] == 1
def test_a_resume_takes_the_run_and_the_chunk_it_replaced_stops(
db_sync, tmp_path, run_env, monkeypatch,
):
from backend.app.tasks import backup as tasks
for c in "ab":
_image(db_sync, tmp_path, c)
row = _going(db_sync, "killed")
def send_then_lose_the_run(target, blob, images_root, pacer):
# While this chunk sends its first file, a newer one takes the run.
with tasks._sync_session_factory()() as s:
other = s.get(BackupRun, row.id)
other.manifest = {**other.manifest, "chunk": "newer"}
s.commit()
return backup_media.blob_key(blob.sha256, blob.path)
monkeypatch.setattr(backup_media, "send", send_then_lose_the_run)
result = tasks.backup_media_task(run_id=row.id, resume=True)
assert "took the run over" in result["skipped"]
assert run_env == []
db_sync.refresh(row)
assert row.status == "running" and row.manifest["chunk"] == "newer"
assert row.manifest["sent"] == 0 and row.manifest["cursor"] == ""
def test_a_restart_resumes_every_media_run_still_going(db_sync, run_env):
from backend.app.tasks.backup import resume_offsite_runs
going = _going(db_sync, "killed")
db_sync.add(BackupRun(kind="media", status="ok", triggered_by="manual",
started_at=datetime.now(UTC), manifest={}))
db_sync.commit()
assert resume_offsite_runs()["media"] == [going.id]
assert run_env == [{"run_id": going.id, "resume": True}]
def test_a_second_run_waits_for_the_one_going(db_sync, run_env):