bvandeusenandClaude Opus 5.5
0fad744bfb
CI and images / lint (push) Successful in 4s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Failing after 31s
CI and images / integration (push) Successful in 2m23s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
A walk that outlives the 90s stop grace is SIGKILLed: its event never finalizes and its platform lock is held for the 27-min TTL. The 30-min sweep then errored every stranded event and bumped consecutive_failures, backing sources off (and blocking backfills) as if the platform failed. On worker_ready, the process consuming 'download' ends pre-boot pending/running events as skipped with error_type 'interrupted', leaves the source untouched so the next tick resumes it, and releases the serialized platforms' locks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
72 lines
2.2 KiB
Python
72 lines
2.2 KiB
Python
"""#4433: only the process consuming `download` clears a restart's leftovers.
|
|
|
|
The consolidated container boots four lanes side by side. If the scheduler or
|
|
ml lane ran this too, a lane restarting on its own (supervisord restarts a
|
|
crashed program) would close the events of walks that are still running.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from backend.app import celery_signals
|
|
|
|
|
|
def _consumer(*queues: str):
|
|
return SimpleNamespace(
|
|
task_consumer=SimpleNamespace(queues=[SimpleNamespace(name=q) for q in queues])
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def calls(monkeypatch):
|
|
seen: list[str] = []
|
|
|
|
class _Session:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def commit(self):
|
|
seen.append("commit")
|
|
|
|
monkeypatch.setattr(celery_signals, "sync_session_factory", lambda: _Session)
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.maintenance.interrupt_orphaned_download_events",
|
|
lambda session, *, booted_at: seen.append("interrupt") or 0,
|
|
)
|
|
monkeypatch.setattr(
|
|
"backend.app.services.platform_lock.release_all_platform_locks",
|
|
lambda: seen.append("release") or 0,
|
|
)
|
|
return seen
|
|
|
|
|
|
def test_the_download_lane_clears_orphans_and_locks(calls):
|
|
celery_signals._on_worker_ready(sender=_consumer("default", "download", "import"))
|
|
assert calls == ["interrupt", "commit", "release"]
|
|
|
|
|
|
@pytest.mark.parametrize("queues", [("maintenance", "scan"), ("ml",), ("maintenance_long",)])
|
|
def test_other_lanes_leave_downloads_alone(calls, queues):
|
|
celery_signals._on_worker_ready(sender=_consumer(*queues))
|
|
assert calls == []
|
|
|
|
|
|
def test_unreadable_queues_do_nothing_rather_than_guess(calls):
|
|
celery_signals._on_worker_ready(sender=SimpleNamespace())
|
|
assert calls == []
|
|
|
|
|
|
def test_a_failure_does_not_stop_the_worker_starting(monkeypatch, calls):
|
|
def boom(session, *, booted_at):
|
|
raise RuntimeError("db down")
|
|
|
|
monkeypatch.setattr(
|
|
"backend.app.tasks.maintenance.interrupt_orphaned_download_events", boom
|
|
)
|
|
celery_signals._on_worker_ready(sender=_consumer("download")) # must not raise
|