fix(external): split fetch timeout into read (60s) + total (30m) budgets (#883)
The single _FETCH_TIMEOUT=3000s meant different things per host: a TOTAL wall-clock for mega (subprocess), but only a per-read socket timeout for HTTP hosts (requests' timeout is the idle gap between bytes, never a total). So a stalled HTTP connection tied up a download-worker slot AND the per-host serialize lock for ~50 min before failing (operator-flagged 2026-06-17). Split into two limits in external_fetch: - read timeout (_READ_TIMEOUT=60s, with _CONNECT_TIMEOUT=30s) → requests gets (connect, read); a stalled socket now fails in ~60s. - total budget (_TOTAL_TIMEOUT=30min) → enforced as a wall-clock deadline across chunks in _stream_to_file (HTTP has no total-download timeout), and passed as the subprocess total for mega. fetch_external() signature: timeout= → read_timeout=/total_timeout=. gdrive (gdown) self-manages; the celery hard limit is the outer backstop. Also lowered the per-host lock TTL 3600→2400 so a worker that dies holding it can't wedge a host's links much past one fetch's budget. Each external link is already one Celery task (sweep enqueues one fetch_external_link.delay per link), so these budgets are per-link. Tests: total-budget-exceeded cleans the .part; HTTP gets (connect, read); mega gets the total. Worker fakes updated to **kwargs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -139,3 +139,43 @@ def test_should_stop_aborts_and_cleans_part(tmp_path, monkeypatch):
|
||||
)
|
||||
assert not res.ok
|
||||
assert list(tmp_path.glob("*.part")) == []
|
||||
|
||||
|
||||
def test_total_budget_exceeded_aborts_and_cleans_part(tmp_path, monkeypatch):
|
||||
# A negative total_timeout puts the deadline in the past, so the first
|
||||
# chunk's budget check trips — simulates a slow trickle blowing the budget.
|
||||
monkeypatch.setattr(ef, "_http_get", lambda *a, **k: _resp(chunks=(b"a", b"b")))
|
||||
res = fetch_external(
|
||||
"pixeldrain", "https://pixeldrain.com/u/x", tmp_path, total_timeout=-1,
|
||||
)
|
||||
assert not res.ok
|
||||
assert "budget" in res.error
|
||||
assert list(tmp_path.glob("*.part")) == []
|
||||
|
||||
|
||||
def test_http_get_uses_connect_and_read_timeout_tuple(tmp_path, monkeypatch):
|
||||
# A stalled socket must fail on the short read timeout, not the total — so
|
||||
# the HTTP layer gets (connect, read), never the total budget.
|
||||
seen = {}
|
||||
|
||||
def fake_get(url, *, timeout, headers=None, stream=True):
|
||||
seen["timeout"] = timeout
|
||||
return _resp()
|
||||
|
||||
monkeypatch.setattr(ef, "_http_get", fake_get)
|
||||
fetch_external("pixeldrain", "https://pixeldrain.com/u/x", tmp_path, read_timeout=42)
|
||||
assert seen["timeout"] == (ef._CONNECT_TIMEOUT, 42)
|
||||
|
||||
|
||||
def test_mega_uses_total_timeout(tmp_path, monkeypatch):
|
||||
# megatools is a subprocess: its only timeout is a total wall-clock, so the
|
||||
# total budget (not the read timeout) applies to it.
|
||||
seen = {}
|
||||
|
||||
def fake_mega(url, out_dir, *, timeout):
|
||||
seen["timeout"] = timeout
|
||||
(Path(out_dir) / "film.mp4").write_bytes(b"vid")
|
||||
|
||||
monkeypatch.setattr(ef, "_run_mega_get", fake_mega)
|
||||
fetch_external("mega", "https://mega.nz/file/x#k", tmp_path, total_timeout=123)
|
||||
assert seen["timeout"] == 123
|
||||
|
||||
Reference in New Issue
Block a user