fix: a restart no longer strands downloads — the download lane closes its predecessor's runs as interrupted and frees the platform locks at boot (#4433)

A walk that outlives the 90s stop grace is SIGKILLed: its event never
finalizes and its platform lock is held for the 27-min TTL. The 30-min
sweep then errored every stranded event and bumped consecutive_failures,
backing sources off (and blocking backfills) as if the platform failed.

On worker_ready, the process consuming 'download' ends pre-boot
pending/running events as skipped with error_type 'interrupted', leaves
the source untouched so the next tick resumes it, and releases the
serialized platforms' locks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

backend/app/tasks/maintenance.py

@@ -7,7 +7,8 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 from PIL import Image
-from sqlalchemy import Integer, and_, cast, delete, func, or_, select, update
+from sqlalchemy import Integer, and_, cast, delete, func, literal, or_, select, update
+from sqlalchemy.dialects.postgresql import JSONB
 
 from ..celery_app import celery
 from ..models import (
@@ -731,6 +732,48 @@ def recover_stalled_download_events() -> int:
         return events_recovered
 
 
+DOWNLOAD_INTERRUPTED_MESSAGE = (
+    "interrupted by a worker restart — the next check picks it up where it left off"
+)
+
+
+def interrupt_orphaned_download_events(session, *, booted_at: datetime) -> int:
+    """Close the download events a restart orphaned, without blaming the source.
+
+    Called when the download lane comes up (#4433). Anything still
+    pending/running from before this boot belongs to the previous process:
+    a walk that outlived the 90s stop grace was SIGKILLed, and a queued or
+    serialize-deferred task is held unacked until Redis redelivers it about an
+    hour later. Left alone, the 30-min stall sweep would error each one and
+    bump `consecutive_failures`, backing the source off as if the platform had
+    failed.
+
+    Instead they end as `skipped` (terminal, not a failure) and the source is
+    not touched: `last_checked_at` keeps its old value, so the next tick finds
+    it due and the walk resumes from its checkpoint. A redelivered message
+    that arrives later finds no pending event and opens a fresh one.
+
+    An event promoted to running after the boot has `started_at` reset to its
+    real start (download_service), so it is never caught here. Does NOT commit.
+    """
+    now = datetime.now(UTC)
+    result = session.execute(
+        update(DownloadEvent)
+        .where(DownloadEvent.status.in_(["pending", "running"]))
+        .where(DownloadEvent.started_at < booted_at)
+        .values(
+            status="skipped",
+            finished_at=now,
+            error=DOWNLOAD_INTERRUPTED_MESSAGE,
+            metadata_=DownloadEvent.metadata_.op("||")(
+                literal({"error_type": "interrupted"}, JSONB)
+            ),
+        )
+        .returning(DownloadEvent.id)
+    )
+    return len(result.all())
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
 def recover_stalled_backup_runs() -> int:
     """Flip BackupRun rows stuck in running/restoring past the hard limit