From d28db32012d88e2d8992db9c3b02162b43586300 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 30 May 2026 02:25:04 -0400 Subject: [PATCH] fix(download): align gallery-dl subprocess timeout to Celery soft limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SourceConfig.timeout defaulted to 3600s (1 hour), but download_source's Celery task has soft_time_limit=900s and hard time_limit=1200s. So gallery-dl never hit its own subprocess.run timeout — Celery always killed it first. The hard SIGKILL leaves no terminal flip on the DownloadEvent, which then sat pending/running until the recovery sweep flipped it to error at 30 min from start. From the operator's seat that was a ~30–40 min "hang" on every retry of a broken source. Pinning the default to 900s (matching Celery's soft_time_limit) lets subprocess.run raise TimeoutExpired cleanly inside Celery's window, and the existing `except subprocess.TimeoutExpired` branch (gallery_dl.py:635) captures it as a clean error_type='timeout' with a real message. The DownloadEvent flips to error in ~15 min instead of waiting on the recovery sweep at 30. Per-source bumps still live in source.config_overrides for legitimately long first-syncs. The new _DEFAULT_GDL_TIMEOUT_SECONDS constant carries the rationale and the Celery-soft-limit dependency in its comment. Operator-flagged 2026-05-30 on 59-source strand pile retries. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/gallery_dl.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index e149477..2883bd4 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -42,6 +42,15 @@ class ErrorType(StrEnum): UNKNOWN_ERROR = "unknown_error" +# Pinned to download_source's Celery soft_time_limit (900s, see +# tasks/download.py:32). Anything larger and Celery kills the task before +# subprocess.run can raise TimeoutExpired — leaving the DownloadEvent +# stranded for the recovery sweep instead of capturing a clean timeout +# error. Per-source bumps live in source.config_overrides for legitimately +# long syncs. Operator-confirmed 2026-05-30 (~40-min hang investigation). +_DEFAULT_GDL_TIMEOUT_SECONDS = 900 + + @dataclass class SourceConfig: content_types: list[str] = field(default_factory=lambda: ["all"]) @@ -51,7 +60,7 @@ class SourceConfig: filename_pattern: str | None = None skip_existing: bool = True save_metadata: bool = True - timeout: int = 3600 + timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS @classmethod def from_dict(cls, data: dict) -> SourceConfig: @@ -63,7 +72,7 @@ class SourceConfig: filename_pattern=data.get("filename_pattern"), skip_existing=data.get("skip_existing", True), save_metadata=data.get("save_metadata", True), - timeout=data.get("timeout", 3600), + timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS), )