fix: a media backup or restore cut off by a restart carries on where it stopped

A media backup and a restore's file phase run as chains of 45-minute chunks,
each queuing the next when it ends. A restart killed the chunk in flight, so
nothing queued the next one; the run's row stayed "running", "Back up now"
skipped it as already running, and only the 7-hour stall sweep ended it.

- When the long-maintenance lane starts, it queues the next chunk of every
  media run still running and every restore fetching its files.
- Each chunk writes its token to the run and hands it to the chunk it queues.
  A chunk queued by any other chunk is dropped (a killed chunk's message
  redelivered by acks_late, say), so a resume never doubles the chain; a chunk
  taken over stops at its next file.
- The integration job installs postgresql-client when the CI image lacks it:
  the backup round trip in 375e7e2 runs the real pg_dump and pg_restore.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/celery_signals.py

@@ -232,17 +232,38 @@ def _consumed_queues(consumer) -> set[str]:
 
 @worker_ready.connect
 def _on_worker_ready(sender=None, **_):
-    """The download lane clears what its previous process left behind (#4433).
+    """A lane picks up what its previous process left behind. Only the lane
+    that runs the work does it; the other lanes booting beside it must not,
+    or a lane restarting on its own would disturb work still running.
+    Best-effort: a failure is logged and the worker still starts."""
+    queues = _consumed_queues(sender)
+    if "download" in queues:
+        _clear_download_leftovers()
+    if "maintenance_long" in queues:
+        _resume_offsite_runs()
+
+
+def _resume_offsite_runs() -> None:
+    """The long-maintenance lane carries on an off-site backup or restore that
+    a restart cut off mid-chunk. With nothing queuing its next chunk, the run
+    would read "running" for hours, and block every new run, until the stall
+    sweep ended it."""
+    try:
+        from .tasks.backup import resume_offsite_runs
+
+        resume_offsite_runs()
+    except Exception:  # noqa: BLE001 — never block the worker from starting
+        log.exception("long-maintenance lane boot: resuming off-site runs failed")
+
+
+def _clear_download_leftovers() -> None:
+    """The download lane clears what its previous process left behind.
 
     A restart SIGKILLs any walk that outlives the stop grace, so its event is
     never finalized and its platform lock is never released. Both used to wait
     out timers — a 30-min stall sweep that then blamed the source, and a 27-min
-    lock TTL that stalled every other source on the platform. Only the process
-    consuming `download` does this; the other lanes booting beside it must not.
-    Best-effort: a failure here is logged and the worker still starts.
+    lock TTL that stalled every other source on the platform.
     """
-    if "download" not in _consumed_queues(sender):
-        return
     booted_at = datetime.now(UTC)
     try:
         from .services.download_recovery import interrupt_orphaned_download_events

backend/app/tasks/backup.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import logging
 import time
+import uuid
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
@@ -371,6 +372,79 @@ def _offsite_manifest(target) -> dict:
             "key": backup_media.PREFIX}
 
 
+# ── One chunk at a time ──────────────────────────────────────────────────────
+#
+# A media backup and a restore's file phase run as a chain of chunks, each
+# queuing the next. The chunk carrying a run writes its token to the run's
+# manifest as "chunk", and hands that token to the chunk it queues as `after`.
+# A chunk queued to resume a run after a restart carries `resume` instead and
+# takes the run over. Any other message is stale (a killed chunk's own message
+# delivered again, say) and is dropped, so a run never goes on twice and the
+# chain never doubles.
+
+
+def _chunk_token(task) -> str:
+    return task.request.id or uuid.uuid4().hex
+
+
+def _claim_run(session, run_id: int, status: str, token: str, *, after: str | None,
+               resume: bool) -> str | None:
+    """Take the run for this chunk. Returns None when taken, or why not."""
+    row = session.get(BackupRun, run_id, with_for_update=True)
+    if row is None or row.status != status:
+        session.rollback()
+        return "the run is no longer going"
+    held = (row.manifest or {}).get("chunk")
+    if not resume and held is not None and held != after:
+        session.rollback()
+        return "another chunk carries this run on"
+    row.manifest = {**(row.manifest or {}), "chunk": token}
+    session.commit()
+    return None
+
+
+def _still_ours(session, run_id: int, token: str) -> bool:
+    """Lock the run's row and say whether this chunk still carries it. A chunk
+    that was taken over stops at its next file rather than write over the
+    newer chunk's place; on False the transaction is already rolled back."""
+    held = session.execute(
+        select(BackupRun.manifest).where(BackupRun.id == run_id).with_for_update()
+    ).scalar_one_or_none()
+    if (held or {}).get("chunk") == token:
+        return True
+    session.rollback()
+    return False
+
+
+def resume_offsite_runs() -> dict:
+    """Queue the next chunk of every off-site run a restart cut short: a media
+    backup still running, and a restore fetching its files. Called when the
+    long-maintenance lane starts, where those chunks run, so none of theirs is
+    alive then; the queued chunk carries on from the run's saved place. A
+    restore stopped while loading its database isn't resumed: the stall sweep
+    ends it, and the screen offers the restore again."""
+    with _sync_session_factory()() as session:
+        media = list(session.execute(
+            select(BackupRun.id).where(BackupRun.kind == "media")
+            .where(BackupRun.status == "running")
+        ).scalars())
+        restores = [
+            r.id for r in session.execute(
+                select(BackupRun).where(BackupRun.kind == "restore")
+                .where(BackupRun.status == "restoring")
+            ).scalars()
+            if (r.manifest or {}).get("phase") == "files"
+        ]
+    for run_id in media:
+        backup_media_task.delay(run_id=run_id, resume=True)
+    for run_id in restores:
+        restore_offsite_task.delay(run_id=run_id, resume=True)
+    if media or restores:
+        log.info("resumed off-site runs after a restart: media %s, restore %s",
+                 media, restores)
+    return {"media": media, "restore": restores}
+
+
 @celery.task(
     name="backend.app.tasks.backup.backup_media_task",
     bind=True,
@@ -381,15 +455,17 @@ def _offsite_manifest(target) -> dict:
     # limit included.
     soft_time_limit=21600, time_limit=23400,
 )
-def backup_media_task(self, *, run_id: int | None = None,
-                      triggered_by: str = "manual") -> dict:
+def backup_media_task(self, *, run_id: int | None = None, triggered_by: str = "manual",
+                      after: str | None = None, resume: bool = False) -> dict:
     """Send the library's images and attachments that aren't in the off-site
     bucket yet, one chunk of a run. Called without `run_id` it starts a run
     (unless one is already going); each chunk queues the next until nothing is
-    left. A file that can't be sent is listed on the run and tried again by
-    the next run. Returns {'backup_run_id': N, ...}."""
+    left, and a restart resumes the run where it stopped. A file that can't be
+    sent is listed on the run and tried again by the next run. Returns
+    {'backup_run_id': N, ...}."""
     SessionLocal = _sync_session_factory()
     now = datetime.now(UTC)
+    token = _chunk_token(self)
     with SessionLocal() as session:
         if run_id is None:
             going = session.execute(
@@ -403,14 +479,15 @@ def backup_media_task(self, *, run_id: int | None = None,
                 return {"skipped": "a restore is going"}
             row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
                             started_at=now, last_progress_at=now, size_bytes=0,
-                            manifest={"sent": 0, "failed": 0, "failures": [], "cursor": ""})
+                            manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
+                                      "chunk": token})
             session.add(row)
             session.commit()
             run_id = row.id
         else:
-            row = session.get(BackupRun, run_id)
-            if row is None or row.status != "running":
-                return {"backup_run_id": run_id, "skipped": "run is no longer running"}
+            why = _claim_run(session, run_id, "running", token, after=after, resume=resume)
+            if why:
+                return {"backup_run_id": run_id, "skipped": why}
 
     try:
         with SessionLocal() as session:
@@ -419,7 +496,8 @@ def backup_media_task(self, *, run_id: int | None = None,
             limit_mbps = settings.backup_upload_limit_mbps
         if target is None:
             raise backup_s3.S3Error("No off-site destination is set up.")
-        more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps))
+        more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
+                                 token)
     except (SoftTimeLimitExceeded, Exception) as exc:
         with SessionLocal() as session:
             row = session.get(BackupRun, run_id)
@@ -427,8 +505,10 @@ def backup_media_task(self, *, run_id: int | None = None,
                 _mark_failed(session, row, exc)
         raise
 
+    if more is None:
+        return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
     if more:
-        backup_media_task.delay(run_id=run_id)
+        backup_media_task.delay(run_id=run_id, after=token)
         return {"backup_run_id": run_id, "continues": True}
     with SessionLocal() as session:
         row = session.get(BackupRun, run_id)
@@ -443,10 +523,11 @@ def backup_media_task(self, *, run_id: int | None = None,
     return {"backup_run_id": run_id, "continues": False}
 
 
-def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
+def _send_media_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
     """Send files until the chunk's time is up or none are left. Progress is
     committed after every file, so a chunk that dies loses at most the file
-    in flight. Returns True when files remain for another chunk."""
+    in flight. Returns True when files remain for another chunk, and None when
+    another chunk has taken the run over."""
     tid = backup_media.target_id(target)
     deadline = time.monotonic() + MEDIA_CHUNK_SECONDS
     with SessionLocal() as session:
@@ -460,6 +541,8 @@ def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
             # across a long transfer holds back vacuum for its whole length.
             session.commit()
             if not batch:
+                if not _still_ours(session, run_id, token):
+                    return None
                 m["cursor"] = cursor
                 row.manifest = dict(m)
                 row.last_progress_at = datetime.now(UTC)
@@ -467,10 +550,13 @@ def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
                 return False
             for blob in batch:
                 if time.monotonic() > deadline:
+                    if not _still_ours(session, run_id, token):
+                        return None
                     m["cursor"] = cursor
                     row.manifest = dict(m)
                     session.commit()
                     return True
+                key = None
                 try:
                     key = backup_media.send(target, blob, IMAGES_ROOT, pacer)
                 except (backup_s3.S3Error, OSError) as exc:
@@ -478,7 +564,9 @@ def _send_media_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
                     if len(m.get("failures") or []) < MEDIA_FAILURES_LISTED:
                         m["failures"] = [*(m.get("failures") or []),
                                          {"path": blob.path, "error": str(exc)[:300]}]
-                else:
+                if not _still_ours(session, run_id, token):
+                    return None
+                if key is not None:
                     backup_media.record(session, tid, [(blob, key)])
                     m["sent"] = m.get("sent", 0) + 1
                     row.size_bytes = (row.size_bytes or 0) + blob.size_bytes
@@ -548,10 +636,15 @@ def _library_is_empty(session) -> bool:
     )).scalar()
 
 
-def _note(session, row: BackupRun, m: dict) -> None:
+def _note(session, row: BackupRun, m: dict, token: str | None = None) -> bool:
+    """Write the run's progress. Given a chunk's token, writes only while that
+    chunk still carries the run, and returns False when it doesn't."""
+    if token is not None and not _still_ours(session, row.id, token):
+        return False
     row.manifest = dict(m)
     row.last_progress_at = datetime.now(UTC)
     session.commit()
+    return True
 
 
 @celery.task(
@@ -561,19 +654,22 @@ def _note(session, row: BackupRun, m: dict) -> None:
     soft_time_limit=21600, time_limit=23400,
 )
 def restore_offsite_task(self, *, dump_key: str | None = None, files_only: bool = False,
-                         run_id: int | None = None) -> dict:
+                         run_id: int | None = None, after: str | None = None,
+                         resume: bool = False) -> dict:
     """Restore this install from the off-site bucket, one chunk at a time.
 
     Started without `run_id`, it replaces the database with the dump at
     `dump_key` (the newest when None), then fetches every library file the
     restored rows name. With `files_only`, the database stays and only the
     missing files are fetched, which also repairs a library that lost files.
-    Each chunk queues the next; the run's row counts files restored from the
-    bucket, copied from a duplicate already on disk, already present, missing
-    from the bucket, and failed. Returns {'backup_run_id': N, ...}."""
+    Each chunk queues the next, and a restart resumes the file phase where it
+    stopped; the run's row counts files restored from the bucket, copied from
+    a duplicate already on disk, already present, missing from the bucket,
+    and failed. Returns {'backup_run_id': N, ...}."""
     SessionLocal = _sync_session_factory()
+    token = _chunk_token(self)
     if run_id is not None:
-        return _restore_files(SessionLocal, run_id)
+        return _restore_files(SessionLocal, run_id, token, after=after, resume=resume)
 
     now = datetime.now(UTC)
     with SessionLocal() as session:
@@ -601,7 +697,7 @@ def restore_offsite_task(self, *, dump_key: str | None = None, files_only: bool
         session.commit()
 
     if files_only:
-        return _restore_files(SessionLocal, run_id)
+        return _restore_files(SessionLocal, run_id, token, after=None, resume=False)
     run_id = _restore_database(SessionLocal, run_id, target, limit_mbps, dump_key)
     # The files follow as chunks of their own, on the worker.
     restore_offsite_task.delay(run_id=run_id)
@@ -706,12 +802,14 @@ def _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, mani
         log.exception("restore: the database couldn't be reset after a failed load")
 
 
-def _restore_files(SessionLocal, run_id: int) -> dict:
+def _restore_files(SessionLocal, run_id: int, token: str, *, after: str | None,
+                   resume: bool) -> dict:
     """One chunk of a restore's file phase; queues the next while files remain."""
     with SessionLocal() as session:
+        why = _claim_run(session, run_id, "restoring", token, after=after, resume=resume)
+        if why:
+            return {"backup_run_id": run_id, "skipped": why}
         row = session.get(BackupRun, run_id)
-        if row is None or row.status != "restoring":
-            return {"backup_run_id": run_id, "skipped": "the restore is no longer going"}
         settings = ImportSettings.load_sync(session)
         limit_mbps = settings.backup_upload_limit_mbps
         try:
@@ -722,7 +820,8 @@ def _restore_files(SessionLocal, run_id: int) -> dict:
             _mark_failed(session, row, exc)
             raise
     try:
-        more = _fetch_files_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps))
+        more = _fetch_files_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
+                                  token)
     except (SoftTimeLimitExceeded, Exception) as exc:
         with SessionLocal() as session:
             row = session.get(BackupRun, run_id)
@@ -730,8 +829,10 @@ def _restore_files(SessionLocal, run_id: int) -> dict:
                 _mark_failed(session, row, exc)
         raise
 
+    if more is None:
+        return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
     if more:
-        restore_offsite_task.delay(run_id=run_id)
+        restore_offsite_task.delay(run_id=run_id, after=token)
         return {"backup_run_id": run_id, "continues": True}
     with SessionLocal() as session:
         row = session.get(BackupRun, run_id)
@@ -751,9 +852,10 @@ def _restore_files(SessionLocal, run_id: int) -> dict:
     return {"backup_run_id": run_id, "continues": False}
 
 
-def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
+def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
     """Put library files back until the chunk's time is up or none are left.
-    Returns True when files remain for another chunk."""
+    Returns True when files remain for another chunk, and None when another
+    chunk has taken the run over."""
     tid = backup_media.target_id(target)
     deadline = time.monotonic() + RESTORE_CHUNK_SECONDS
     with SessionLocal() as session:
@@ -768,13 +870,11 @@ def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
             session.commit()
             if not batch:
                 m["cursor"] = cursor
-                _note(session, row, m)
-                return False
+                return False if _note(session, row, m, token) else None
             for f in batch:
                 if time.monotonic() > deadline:
                     m["cursor"] = cursor
-                    _note(session, row, m)
-                    return True
+                    return True if _note(session, row, m, token) else None
                 try:
                     outcome = backup_restore.restore_file(
                         target, f, keys.get(f.sha256), IMAGES_ROOT, pacer)
@@ -789,5 +889,6 @@ def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer) -> bool:
                 cursor = f.sha256
                 m["cursor"] = cursor
                 if outcome != "present" or time.monotonic() - noted > _RESTORE_NOTE_SECONDS:
-                    _note(session, row, m)
+                    if not _note(session, row, m, token):
+                        return None
                     noted = time.monotonic()