fix(external): split fetch timeout into read (60s) + total (30m) budgets (#883)

The single _FETCH_TIMEOUT=3000s meant different things per host: a TOTAL
wall-clock for mega (subprocess), but only a per-read socket timeout for HTTP
hosts (requests' timeout is the idle gap between bytes, never a total). So a
stalled HTTP connection tied up a download-worker slot AND the per-host
serialize lock for ~50 min before failing (operator-flagged 2026-06-17).

Split into two limits in external_fetch:
- read timeout (_READ_TIMEOUT=60s, with _CONNECT_TIMEOUT=30s) → requests gets
  (connect, read); a stalled socket now fails in ~60s.
- total budget (_TOTAL_TIMEOUT=30min) → enforced as a wall-clock deadline
  across chunks in _stream_to_file (HTTP has no total-download timeout), and
  passed as the subprocess total for mega.
fetch_external() signature: timeout= → read_timeout=/total_timeout=. gdrive
(gdown) self-manages; the celery hard limit is the outer backstop.

Also lowered the per-host lock TTL 3600→2400 so a worker that dies holding it
can't wedge a host's links much past one fetch's budget.

Each external link is already one Celery task (sweep enqueues one
fetch_external_link.delay per link), so these budgets are per-link.

Tests: total-budget-exceeded cleans the .part; HTTP gets (connect, read);
mega gets the total. Worker fakes updated to **kwargs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/external_fetch.py

@@ -9,7 +9,8 @@ call `fetch_external()` directly.
 No single tool covers all five hosts, so a small registry maps host → fetch
 function behind one signature:
 
-    fetch_external(host, url, dest_dir, *, timeout, should_stop) -> FetchResult
+    fetch_external(host, url, dest_dir, *, read_timeout, total_timeout, should_stop)
+        -> FetchResult
 
 Backends:
   - dropbox    : force the direct-download variant (dl=1) + stream GET.
@@ -30,6 +31,7 @@ import logging
 import os
 import re
 import subprocess
+import time
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -40,7 +42,19 @@ import requests
 log = logging.getLogger(__name__)
 
 _CHUNK = 1 << 16
-_DEFAULT_TIMEOUT = 600.0
+# Two distinct limits, because conflating them (the old single 3000s value) meant
+# a stalled HTTP connection tied up a download-worker slot + the per-host lock for
+# ~50 min before failing (operator-flagged 2026-06-17):
+#   * READ timeout — max idle gap between bytes on an HTTP socket. A stalled host
+#     (socket open, nothing flowing) is the common failure mode; a short read
+#     timeout fails it fast. This is what requests' `timeout` actually enforces —
+#     per-read, never a total.
+#   * TOTAL budget — generous wall-clock cap for a file that IS actively
+#     transferring (big films/packs). Enforced as a deadline across chunks, since
+#     no HTTP client timeout bounds the total. Also the subprocess total for mega.
+_CONNECT_TIMEOUT = 30.0
+_READ_TIMEOUT = 60.0
+_TOTAL_TIMEOUT = 1800.0  # 30 min per fetch
 _USER_AGENT = (
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -122,9 +136,11 @@ def _filename_from(resp: requests.Response, url: str, fallback: str) -> str:
 
 
 def _stream_to_file(resp: requests.Response, dest: Path,
-                    should_stop: Callable[[], bool]) -> int:
+                    should_stop: Callable[[], bool],
+                    *, deadline: float | None = None) -> int:
     """Stream a response body to `dest` (atomic via .part). Returns byte count.
-    Honors should_stop between chunks (partial file removed)."""
+    Honors should_stop and the total-budget `deadline` (a time.monotonic() value)
+    between chunks; the partial file is removed on either abort."""
     part = dest.with_name(dest.name + ".part")
     total = 0
     try:
@@ -132,6 +148,8 @@ def _stream_to_file(resp: requests.Response, dest: Path,
             for chunk in resp.iter_content(chunk_size=_CHUNK):
                 if should_stop():
                     raise ExternalFetchError("stopped")
+                if deadline is not None and time.monotonic() > deadline:
+                    raise ExternalFetchError("exceeded total fetch budget")
                 if chunk:
                     fh.write(chunk)
                     total += len(chunk)
@@ -142,21 +160,29 @@ def _stream_to_file(resp: requests.Response, dest: Path,
     return total
 
 
-def _get_to_dir(url: str, dest_dir: Path, *, timeout: float,
-                should_stop: Callable[[], bool], fallback: str,
-                headers: dict | None = None) -> FetchResult:
-    resp = _http_get(url, timeout=timeout, headers=headers, stream=True)
+def _get_to_dir(url: str, dest_dir: Path, *, read_timeout: float,
+                total_timeout: float, should_stop: Callable[[], bool],
+                fallback: str, headers: dict | None = None) -> FetchResult:
+    # (connect, read): a short read timeout fails a stalled socket fast; the total
+    # budget is enforced separately as a deadline across chunks (requests has no
+    # total-download timeout).
+    resp = _http_get(
+        url, timeout=(_CONNECT_TIMEOUT, read_timeout), headers=headers, stream=True
+    )
     if resp.status_code != 200:
         return FetchResult(error=f"HTTP {resp.status_code} for {url}")
     name = _filename_from(resp, url, fallback)
     dest = dest_dir / name
-    written = _stream_to_file(resp, dest, should_stop)
+    written = _stream_to_file(
+        resp, dest, should_stop, deadline=time.monotonic() + total_timeout
+    )
     return FetchResult(files=[dest], bytes=written)
 
 
 # -- per-host fetchers -----------------------------------------------------
 
-def _fetch_dropbox(url: str, dest_dir: Path, *, timeout: float,
+def _fetch_dropbox(url: str, dest_dir: Path, *, read_timeout: float,
+                   total_timeout: float,
                    should_stop: Callable[[], bool]) -> FetchResult:
     # Force the direct-download variant: dl=1 (Dropbox serves an HTML preview
     # for dl=0). Rewrite/insert the param rather than string-replace so ?dl=0,
@@ -165,35 +191,44 @@ def _fetch_dropbox(url: str, dest_dir: Path, *, timeout: float,
     q = dict(parse_qsl(parts.query))
     q["dl"] = "1"
     direct = urlunsplit(parts._replace(query=urlencode(q)))
-    return _get_to_dir(direct, dest_dir, timeout=timeout,
-                       should_stop=should_stop, fallback="dropbox-file")
+    return _get_to_dir(direct, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback="dropbox-file")
 
 
-def _fetch_pixeldrain(url: str, dest_dir: Path, *, timeout: float,
+def _fetch_pixeldrain(url: str, dest_dir: Path, *, read_timeout: float,
+                      total_timeout: float,
                       should_stop: Callable[[], bool]) -> FetchResult:
     # /u/{id} (and /l/{id}) → the API file endpoint.
     file_id = urlsplit(url).path.rstrip("/").split("/")[-1]
     if not file_id:
         return FetchResult(error=f"no pixeldrain id in {url}")
     api = f"https://pixeldrain.com/api/file/{file_id}"
-    return _get_to_dir(api, dest_dir, timeout=timeout,
-                       should_stop=should_stop, fallback=f"{file_id}.bin")
+    return _get_to_dir(api, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback=f"{file_id}.bin")
 
 
-def _fetch_mediafire(url: str, dest_dir: Path, *, timeout: float,
+def _fetch_mediafire(url: str, dest_dir: Path, *, read_timeout: float,
+                     total_timeout: float,
                      should_stop: Callable[[], bool]) -> FetchResult:
-    page = _http_get(url, timeout=timeout, stream=False)
+    page = _http_get(url, timeout=(_CONNECT_TIMEOUT, read_timeout), stream=False)
     if page.status_code != 200:
         return FetchResult(error=f"HTTP {page.status_code} for mediafire page")
     m = _MEDIAFIRE_RE.search(page.text or "")
     if not m:
         return FetchResult(error="mediafire direct link not found on page")
-    return _get_to_dir(m.group(1), dest_dir, timeout=timeout,
-                       should_stop=should_stop, fallback="mediafire-file")
+    return _get_to_dir(m.group(1), dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback="mediafire-file")
 
 
-def _fetch_gdrive(url: str, dest_dir: Path, *, timeout: float,
+def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
+                  total_timeout: float,
                   should_stop: Callable[[], bool]) -> FetchResult:
+    # gdown manages its own HTTP session/timeouts; the task's celery hard limit is
+    # the outer backstop. read_timeout/total_timeout are accepted for a uniform
+    # registry signature but not separately enforceable here.
     out = _gdown_download(url, str(dest_dir))
     if not out:
         return FetchResult(error="gdown returned no file (quota / private?)")
@@ -203,10 +238,13 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, timeout: float,
     return FetchResult(files=[p], bytes=p.stat().st_size)
 
 
-def _fetch_mega(url: str, dest_dir: Path, *, timeout: float,
+def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
+                total_timeout: float,
                 should_stop: Callable[[], bool]) -> FetchResult:
     before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
-    _run_mega_get(url, str(dest_dir), timeout=timeout)
+    # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
+    # timeout has no analogue here), so the total budget applies directly.
+    _run_mega_get(url, str(dest_dir), timeout=total_timeout)
     new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
     if not new:
         return FetchResult(error="mega-get wrote no new file")
@@ -225,18 +263,24 @@ SUPPORTED_HOSTS = tuple(_REGISTRY)
 
 
 def fetch_external(host: str, url: str, dest_dir: Path, *,
-                   timeout: float = _DEFAULT_TIMEOUT,
+                   read_timeout: float = _READ_TIMEOUT,
+                   total_timeout: float = _TOTAL_TIMEOUT,
                    should_stop: Callable[[], bool] = lambda: False) -> FetchResult:
     """Fetch `url` (a `host` link) into `dest_dir`. Returns a FetchResult; never
-    raises — any backend error (transport, non-200, scrape miss, subprocess
-    failure, stop) is captured on `.error` so the worker can record it and move
-    on."""
+    raises — any backend error (transport, read/total timeout, non-200, scrape
+    miss, subprocess failure, stop) is captured on `.error` so the worker can
+    record it and move on.
+
+    `read_timeout` fails a stalled HTTP socket fast (idle gap between bytes);
+    `total_timeout` is the generous wall-clock cap for a large file that is
+    actively transferring (and the subprocess total for mega)."""
     fetcher = _REGISTRY.get(host)
     if fetcher is None:
         return FetchResult(error=f"unsupported host {host!r}")
     dest_dir.mkdir(parents=True, exist_ok=True)
     try:
-        return fetcher(url, dest_dir, timeout=timeout, should_stop=should_stop)
+        return fetcher(url, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop)
     except requests.RequestException as exc:
         return FetchResult(error=f"transport error: {exc}")
     except subprocess.TimeoutExpired: