bvandeusen
66f19d67f5
Three coupled operator-reported pains from the 2026-05-31 download event audit: 1. `[patreon][warning] Not allowed to view post N` was bubbling up as an error event, bumping consecutive_failures and parking the source in "needs attention." The classifier's tier-gated branch was gated on `return_code in (1, 4)`. Gallery-dl returns a different exit code for mixed-failure runs (e.g. paywall warnings + a missing yt-dlp dep flipping the exit bits), so the branch never fired and the path fell through to UNKNOWN_ERROR. Widen the gate: when no source-level error fired AND tier-gated warnings are present, classify as TIER_LIMITED regardless of return code. 2. Knuxy event #38275 (2026-05-31) ran 30 min and finalized with "stranded by recovery sweep (no terminal status after time_limit)" + empty stdout/stderr. Root cause: subprocess.run timeout (900s) and Celery soft_time_limit (900s) raced; when Celery won, SIGKILL wiped the in-memory captured output and the DownloadEvent ended up empty-logged 18 minutes later when the sweep finalized it. Drop gallery-dl's default subprocess timeout to 870s — a 30s margin shy of Celery's soft limit — so subprocess.TimeoutExpired always wins the race and captures the partial stdout/stderr via the existing handler. 3. `[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl` was firing on every video attachment, causing per-item download failures that masked legitimate tier-gated classification. Add yt-dlp>=2025.1 to requirements.txt. Once it's in the image, video posts download normally and the per-item failure noise disappears. Tests added: - pure tier-gated stderr with exit code 128 → TIER_LIMITED + success - mixed tier-gated + yt-dlp + per-item failures → still TIER_LIMITED
252 lines
9.1 KiB
Python
252 lines
9.1 KiB
Python
"""GalleryDLService tests — subprocess is mocked so no real gallery-dl runs."""
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from backend.app.services.gallery_dl import (
|
|
ErrorType,
|
|
GalleryDLService,
|
|
SourceConfig,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def gdl(tmp_path):
|
|
return GalleryDLService(
|
|
images_root=tmp_path / "images",
|
|
rate_limit=3.0,
|
|
validate_files=False,
|
|
)
|
|
|
|
|
|
def _proc(stdout="", stderr="", returncode=0):
|
|
return SimpleNamespace(stdout=stdout, stderr=stderr, returncode=returncode)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_happy_path(gdl, monkeypatch):
|
|
out_path = gdl.images_root / "alice" / "patreon" / "post1" / "img.jpg"
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
out_path.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 32 + b"\xff\xd9")
|
|
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(stdout=str(out_path) + "\n", returncode=0),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice",
|
|
artist_slug="alice",
|
|
platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.success is True
|
|
assert result.files_downloaded == 1
|
|
assert result.error_type is None
|
|
assert str(out_path) in result.written_paths
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_no_new_content(gdl, monkeypatch):
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(stdout="# /images/alice/patreon/old.jpg\n", returncode=0),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.success is True
|
|
assert result.files_downloaded == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_auth_error_categorization(gdl, monkeypatch):
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(
|
|
stdout="", stderr='[patreon][error] "GET /api/foo HTTP/1.1" 401 None\n',
|
|
returncode=1,
|
|
),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.success is False
|
|
assert result.error_type == ErrorType.AUTH_ERROR
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_timeout(gdl, monkeypatch):
|
|
import subprocess as sp
|
|
|
|
def _raise(*a, **k):
|
|
raise sp.TimeoutExpired(cmd="gallery-dl", timeout=10)
|
|
|
|
monkeypatch.setattr("backend.app.services.gallery_dl.subprocess.run", _raise)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(timeout=10),
|
|
)
|
|
assert result.success is False
|
|
assert result.error_type == ErrorType.TIMEOUT
|
|
# Empty-partial timeout: no preserved output and zero files.
|
|
assert result.stdout == ""
|
|
assert result.stderr == ""
|
|
assert result.files_downloaded == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_timeout_preserves_partial_output_and_classifies(gdl, monkeypatch):
|
|
"""When gallery-dl is killed by timeout but had emitted partial output,
|
|
the DownloadResult preserves stdout/stderr, counts files written so
|
|
far, and promotes to RATE_LIMITED if the partial stderr shows a rate-
|
|
limit pattern (so the platform cooldown kicks in). Without this, the
|
|
operator only sees 'timed out' with no clue whether it was 'lots of
|
|
content', 'stuck retrying', or 'hung silent'."""
|
|
import subprocess as sp
|
|
|
|
# Simulate gallery-dl writing two files, then spinning on 429s.
|
|
partial_stdout = (
|
|
"/tmp/images/alice/file_001.jpg\n"
|
|
"/tmp/images/alice/file_002.jpg\n"
|
|
)
|
|
partial_stderr = (
|
|
"[urllib3] 429 Too Many Requests; sleeping 60s\n"
|
|
"[urllib3] 429 Too Many Requests; sleeping 60s\n"
|
|
)
|
|
|
|
def _raise(*a, **k):
|
|
raise sp.TimeoutExpired(
|
|
cmd="gallery-dl", timeout=900,
|
|
output=partial_stdout, stderr=partial_stderr,
|
|
)
|
|
|
|
monkeypatch.setattr("backend.app.services.gallery_dl.subprocess.run", _raise)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(timeout=900),
|
|
)
|
|
assert result.success is False
|
|
# Partial output preserved on the DownloadResult — _phase3_persist
|
|
# writes them into download_event.metadata so the UI can render them.
|
|
assert result.stdout == partial_stdout
|
|
assert result.stderr == partial_stderr
|
|
# Files counted from partial stdout (lines starting with '/').
|
|
assert result.files_downloaded == 2
|
|
# Rate-limit pattern in partial stderr → promoted to RATE_LIMITED so
|
|
# _update_source_health stamps the platform cooldown.
|
|
assert result.error_type == ErrorType.RATE_LIMITED
|
|
assert "Rate-limited" in result.error_message
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_validation_quarantines_truncated_files(tmp_path, monkeypatch):
|
|
gdl = GalleryDLService(
|
|
images_root=tmp_path / "images", rate_limit=3.0, validate_files=True,
|
|
)
|
|
bad = gdl.images_root / "alice" / "patreon" / "post" / "bad.jpg"
|
|
bad.parent.mkdir(parents=True, exist_ok=True)
|
|
bad.write_bytes(b"\xff\xd8\xff" + b"\x00" * 100) # SOI but no EOI
|
|
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(stdout=str(bad) + "\n", returncode=0),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.files_quarantined == 1
|
|
assert len(result.quarantined_paths) == 1
|
|
assert not bad.exists()
|
|
assert result.error_type == ErrorType.VALIDATION_FAILED
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_tier_limited_classified_as_success(gdl, monkeypatch):
|
|
"""Pure tier-gated stderr (paywall warnings only, no source-level
|
|
errors). Should be TIER_LIMITED with success=True so the
|
|
DownloadEvent doesn't surface as 'needs attention' for an operator
|
|
who's just not paying the top tier. Pre-fix: this fell through to
|
|
UNKNOWN_ERROR when the return code wasn't 1 or 4."""
|
|
stderr = (
|
|
"[patreon][warning] Not allowed to view post 159421258\n"
|
|
"[patreon][warning] Not allowed to view post 159247879\n"
|
|
"[patreon][warning] Not allowed to view post 158111214\n"
|
|
)
|
|
# Use a non-(1,4) return code to prove the gate widened.
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(stdout="", stderr=stderr, returncode=128),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.error_type == ErrorType.TIER_LIMITED
|
|
assert result.success is True
|
|
assert "3 posts" in (result.error_message or "")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_tier_limited_with_per_item_video_failures(gdl, monkeypatch):
|
|
"""Real operator log shape (2026-05-31): hundreds of paywall warnings
|
|
+ a handful of per-item video errors caused by missing yt-dlp. The
|
|
video errors are classified as per-item (not source-level), so the
|
|
classifier should still pick TIER_LIMITED. Once yt-dlp is installed
|
|
the video errors stop firing; until then, the source-health doesn't
|
|
flap to 'needs attention' over paywalled content."""
|
|
stderr = (
|
|
"[patreon][warning] Not allowed to view post 159421258\n"
|
|
"[patreon][warning] Not allowed to view post 159247879\n"
|
|
"[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl\n"
|
|
"[download][error] Failed to download 02_video.mp4\n"
|
|
)
|
|
monkeypatch.setattr(
|
|
"backend.app.services.gallery_dl.subprocess.run",
|
|
lambda *a, **k: _proc(stdout="", stderr=stderr, returncode=1),
|
|
)
|
|
result = await gdl.download(
|
|
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
|
source_config=SourceConfig(),
|
|
)
|
|
assert result.error_type == ErrorType.TIER_LIMITED
|
|
assert result.success is True
|
|
|
|
|
|
def test_compute_run_stats(gdl):
|
|
stats = gdl._compute_run_stats(
|
|
return_code=0,
|
|
stdout="/path/a.jpg\n/path/b.jpg\n# /path/c.jpg\n",
|
|
stderr="[patreon][warning] foo\n[patreon][download][error] failed to download /x\n",
|
|
)
|
|
assert stats["downloaded_count"] == 2
|
|
assert stats["skipped_count"] == 1
|
|
assert stats["warning_count"] >= 1
|
|
assert stats["per_item_failures"] == 1
|
|
|
|
|
|
def test_extract_errors_warnings(gdl):
|
|
stderr = (
|
|
"[patreon][debug] foo\n[patreon][error] bar\n"
|
|
"[urllib3][debug] baz\n[patreon][warning] qux\n"
|
|
)
|
|
result = gdl._extract_errors_warnings(stderr)
|
|
assert "[error]" in result
|
|
assert "[warning]" in result
|
|
assert "[debug]" not in result
|
|
|
|
|
|
def test_truncate_log_caps_large_text(gdl):
|
|
huge = "line\n" * 200_000
|
|
result = gdl._truncate_log(huge, max_bytes=10_000)
|
|
assert len(result.encode()) < 11_000
|
|
assert "lines elided" in result
|
|
|
|
|
|
def test_truncate_log_passes_short_text(gdl):
|
|
text = "small\n"
|
|
assert gdl._truncate_log(text) == text
|