fix: a restart no longer strands downloads — the download lane closes its predecessor's runs as interrupted and frees the platform locks at boot (#4433)
CI and images / lint (push) Successful in 4s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Failing after 31s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
CI and images / lint (push) Successful in 4s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Failing after 31s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
A walk that outlives the 90s stop grace is SIGKILLed: its event never finalizes and its platform lock is held for the 27-min TTL. The 30-min sweep then errored every stranded event and bumped consecutive_failures, backing sources off (and blocking backfills) as if the platform failed. On worker_ready, the process consuming 'download' ends pre-boot pending/running events as skipped with error_type 'interrupted', leaves the source untouched so the next tick resumes it, and releases the serialized platforms' locks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
@@ -718,6 +718,57 @@ def test_recover_stalled_download_skips_fresh(db_sync):
|
||||
assert failures == 0
|
||||
|
||||
|
||||
def test_download_lane_boot_closes_pre_boot_events_without_blaming_the_source(db_sync):
|
||||
"""#4433: a restart SIGKILLs a walk past its stop grace and strands queued
|
||||
ones. At the download lane's boot, everything pending/running from before
|
||||
it ends as `skipped`/interrupted — and the source is left as it was, so it
|
||||
is due on the next tick instead of backed off as a failure."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import DownloadEvent, Source
|
||||
from backend.app.tasks.maintenance import (
|
||||
DOWNLOAD_INTERRUPTED_MESSAGE,
|
||||
interrupt_orphaned_download_events,
|
||||
)
|
||||
|
||||
sid = _make_source(db_sync, slug="rebooted")
|
||||
booted_at = datetime.now(UTC)
|
||||
before = booted_at - timedelta(minutes=5)
|
||||
walking = DownloadEvent(
|
||||
source_id=sid, status="running", started_at=before,
|
||||
metadata_={"live": {"downloaded": 3}},
|
||||
)
|
||||
queued = DownloadEvent(source_id=sid, status="pending", started_at=before)
|
||||
finished = DownloadEvent(source_id=sid, status="ok", started_at=before)
|
||||
# Promoted to running after the boot: download_service resets started_at.
|
||||
fresh = DownloadEvent(
|
||||
source_id=sid, status="running", started_at=booted_at + timedelta(seconds=5),
|
||||
)
|
||||
db_sync.add_all([walking, queued, finished, fresh])
|
||||
db_sync.commit()
|
||||
|
||||
closed = interrupt_orphaned_download_events(db_sync, booted_at=booted_at)
|
||||
db_sync.commit()
|
||||
|
||||
assert closed == 2
|
||||
db_sync.expire_all()
|
||||
for ev in (walking, queued):
|
||||
assert ev.status == "skipped"
|
||||
assert ev.error == DOWNLOAD_INTERRUPTED_MESSAGE
|
||||
assert ev.finished_at is not None
|
||||
assert ev.metadata_["error_type"] == "interrupted"
|
||||
assert walking.metadata_["live"] == {"downloaded": 3}
|
||||
assert finished.status == "ok"
|
||||
assert fresh.status == "running"
|
||||
src = db_sync.execute(
|
||||
select(Source.consecutive_failures, Source.last_error, Source.last_checked_at)
|
||||
.where(Source.id == sid)
|
||||
).one()
|
||||
assert src.consecutive_failures == 0
|
||||
assert src.last_error is None
|
||||
assert src.last_checked_at is None
|
||||
|
||||
|
||||
def test_recover_stalled_download_flips_stale_pending(db_sync):
|
||||
"""A 2-hour-old pending event flips to error AND the source is bumped
|
||||
(consecutive_failures, last_error, last_checked_at) so the next scan
|
||||
|
||||
Reference in New Issue
Block a user