diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index c356908..3e19a69 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -42,13 +42,16 @@ class ErrorType(StrEnum): UNKNOWN_ERROR = "unknown_error" -# Pinned to download_source's Celery soft_time_limit (900s, see -# tasks/download.py:32). Anything larger and Celery kills the task before -# subprocess.run can raise TimeoutExpired — leaving the DownloadEvent -# stranded for the recovery sweep instead of capturing a clean timeout -# error. Per-source bumps live in source.config_overrides for legitimately -# long syncs. Operator-confirmed 2026-05-30 (~40-min hang investigation). -_DEFAULT_GDL_TIMEOUT_SECONDS = 900 +# 30 seconds shy of download_source's Celery soft_time_limit (900s, see +# tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before +# Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race, +# SIGKILLs the worker, in-memory stdout/stderr is lost, and the +# DownloadEvent ends up empty-logged with "stranded by recovery sweep" +# 18 minutes later (operator-flagged 2026-05-31, Knuxy event #38275). +# The 30s buffer absorbs scheduler jitter / GC pauses without making +# legitimately-long-running syncs timeout-friendlier. Per-source bumps +# still live in source.config_overrides for legitimately long syncs. +_DEFAULT_GDL_TIMEOUT_SECONDS = 870 @dataclass @@ -369,7 +372,17 @@ class GalleryDLService: if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: return ErrorType.NO_NEW_CONTENT, "No new content to download" - if return_code in (1, 4) and not has_actual_error: + # Tier-gated classification used to require `return_code in (1, 4)`, + # which silently fell through to UNKNOWN_ERROR when gallery-dl + # returned a different exit code for mixed-failure runs (e.g. + # paywall warnings + a missing yt-dlp dep flipping the exit bits). + # The artist then surfaced as "needs attention" purely because a + # paywall blocked posts the operator wasn't paying to see — + # operator-flagged 2026-05-31. Now: if no source-level error + # category fired AND tier-gated warnings are present, classify + # as TIER_LIMITED regardless of return code. Same priority order + # as before (auth/rate/access/not_found/network/http still win). + if not has_actual_error: tier_gated_lines = [ line for line in combined.split("\n") if "][warning]" in line and "not allowed to view post" in line diff --git a/requirements.txt b/requirements.txt index a6e009b..dc76496 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,11 @@ imagehash>=4.3,<4.4 # Gallery-dl wrapper (lands in FC-3) gallery-dl>=1.32,<1.33 +# Video extractor backend for gallery-dl. Without it, every video post +# attachment fails with `[downloader.ytdl][error] Cannot import yt-dlp` +# → `[download][error] Failed to download NN_video.mp4`. Operator-flagged +# 2026-05-31 after Patreon video posts produced empty downloads. +yt-dlp>=2025.1 # Utilities python-dotenv>=1.2,<2.0 diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index 35d2d04..af490d1 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -164,6 +164,58 @@ async def test_validation_quarantines_truncated_files(tmp_path, monkeypatch): assert result.error_type == ErrorType.VALIDATION_FAILED +@pytest.mark.asyncio +async def test_download_tier_limited_classified_as_success(gdl, monkeypatch): + """Pure tier-gated stderr (paywall warnings only, no source-level + errors). Should be TIER_LIMITED with success=True so the + DownloadEvent doesn't surface as 'needs attention' for an operator + who's just not paying the top tier. Pre-fix: this fell through to + UNKNOWN_ERROR when the return code wasn't 1 or 4.""" + stderr = ( + "[patreon][warning] Not allowed to view post 159421258\n" + "[patreon][warning] Not allowed to view post 159247879\n" + "[patreon][warning] Not allowed to view post 158111214\n" + ) + # Use a non-(1,4) return code to prove the gate widened. + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout="", stderr=stderr, returncode=128), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.error_type == ErrorType.TIER_LIMITED + assert result.success is True + assert "3 posts" in (result.error_message or "") + + +@pytest.mark.asyncio +async def test_download_tier_limited_with_per_item_video_failures(gdl, monkeypatch): + """Real operator log shape (2026-05-31): hundreds of paywall warnings + + a handful of per-item video errors caused by missing yt-dlp. The + video errors are classified as per-item (not source-level), so the + classifier should still pick TIER_LIMITED. Once yt-dlp is installed + the video errors stop firing; until then, the source-health doesn't + flap to 'needs attention' over paywalled content.""" + stderr = ( + "[patreon][warning] Not allowed to view post 159421258\n" + "[patreon][warning] Not allowed to view post 159247879\n" + "[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl\n" + "[download][error] Failed to download 02_video.mp4\n" + ) + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout="", stderr=stderr, returncode=1), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.error_type == ErrorType.TIER_LIMITED + assert result.success is True + + def test_compute_run_stats(gdl): stats = gdl._compute_run_stats( return_code=0,