feat(downloader): tier-limited classification, config defaults, yt-dlp support
- Classify Patreon "Not allowed to view post" warnings as TIER_LIMITED so subscription-gated runs are distinguished from genuine failures, and persist error_type/message on completed runs so the distinction survives to the UI. - Fold PLATFORM_DEFAULTS into _get_default_config so gallery-dl.conf is a complete, editable document; _build_config_for_source now preserves the user's conf and only re-seeds missing platform sections. - Add Reset-to-Defaults action in Settings (vs. Revert Changes which just reloads disk); show info banner when no gallery-dl.conf exists yet. - Fix Discord embeds option: must be "all" string, not bool (gallery-dl iterates the value). - Install yt-dlp + ffmpeg in Docker images so Patreon/Mux HLS video posts download instead of logging "Cannot import yt-dlp" and skipping. - Recognize yt-dlp import failures as per-item errors so they don't mask TIER_LIMITED/NO_NEW_CONTENT classification. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -93,3 +93,94 @@ def test_patreon_per_item_download_failure_without_skips_still_fails():
|
||||
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||
|
||||
assert error_type != ErrorType.NO_NEW_CONTENT
|
||||
|
||||
|
||||
def test_patreon_tier_gated_warnings_classify_as_tier_limited():
|
||||
"""Many 'Not allowed to view post N' warnings + exit 4 + no downloads →
|
||||
TIER_LIMITED. Reflects a Patreon tier downgrade where older posts are now
|
||||
paywalled but we don't want to call this a failure."""
|
||||
service = _make_service()
|
||||
stderr = "\n".join(
|
||||
f"[patreon][warning] Not allowed to view post {100000 + i}"
|
||||
for i in range(15)
|
||||
)
|
||||
stdout = ""
|
||||
|
||||
error_type, message = service._categorize_error(4, stdout, stderr)
|
||||
|
||||
assert error_type == ErrorType.TIER_LIMITED
|
||||
assert "15" in message
|
||||
|
||||
|
||||
def test_tier_limited_with_exit_code_1_also_classifies():
|
||||
"""Mixed run: exit 1 with tier-gated warnings and no real errors."""
|
||||
service = _make_service()
|
||||
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||
stdout = ""
|
||||
|
||||
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||
|
||||
assert error_type == ErrorType.TIER_LIMITED
|
||||
|
||||
|
||||
def test_tier_limited_not_applied_when_real_error_present():
|
||||
"""Tier warnings + a real auth error → auth error wins (TIER_LIMITED is
|
||||
only for 'everything else was fine' runs)."""
|
||||
service = _make_service()
|
||||
stderr = (
|
||||
"[patreon][warning] Not allowed to view post 999\n"
|
||||
'[patreon][error] "GET /api/posts HTTP/1.1" 401 None\n'
|
||||
)
|
||||
stdout = ""
|
||||
|
||||
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||
|
||||
assert error_type == ErrorType.AUTH_ERROR
|
||||
|
||||
|
||||
def test_tier_limited_does_not_fire_on_exit_0():
|
||||
"""Successful run with some paywalled posts as warnings isn't an error at
|
||||
all — don't reach the TIER_LIMITED branch (exit 0 → success path earlier)."""
|
||||
service = _make_service()
|
||||
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||
stdout = "/data/downloads/Artist/patreon/post1/img.jpg\n"
|
||||
|
||||
error_type, _message = service._categorize_error(0, stdout, stderr)
|
||||
|
||||
# Exit 0 + stdout activity should not hit the TIER_LIMITED branch
|
||||
assert error_type != ErrorType.TIER_LIMITED
|
||||
|
||||
|
||||
def test_skip_activity_takes_precedence_over_tier_limited():
|
||||
"""If there's also skip activity, NO_NEW_CONTENT wins — precedence matters
|
||||
because the tier_limited branch is the last informational fallback."""
|
||||
service = _make_service()
|
||||
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||
stdout = "# /data/downloads/Artist/patreon/post1/img.jpg\n"
|
||||
|
||||
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||
|
||||
assert error_type == ErrorType.NO_NEW_CONTENT
|
||||
|
||||
|
||||
def test_ytdl_missing_with_skips_classifies_as_no_new_content():
|
||||
"""Real-world scenario from a Patreon source: container is missing yt-dlp
|
||||
so every video logs '[downloader.ytdl][error] Cannot import yt-dlp' plus
|
||||
one [download][error] per video. With overwhelming skip activity, this
|
||||
should classify as NO_NEW_CONTENT (the setup gap is per-item — it
|
||||
shouldn't mask the dominant signal that the archive is up-to-date).
|
||||
|
||||
Regression: previously the yt-dlp import error didn't match any per-item
|
||||
pattern, so has_actual_error stayed True and the run was misclassified as
|
||||
UNKNOWN_ERROR even though 1300+ files were skipped."""
|
||||
service = _make_service()
|
||||
stderr = (
|
||||
"[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl\n"
|
||||
"[download][error] Failed to download 02_video.mp4\n"
|
||||
"[download][error] Failed to download 02_video.mp4\n"
|
||||
)
|
||||
stdout = "# /data/downloads/Artist/patreon/post/01_img.jpg\n" * 100
|
||||
|
||||
error_type, _message = service._categorize_error(4, stdout, stderr)
|
||||
|
||||
assert error_type == ErrorType.NO_NEW_CONTENT
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Unit tests for the unified gallery-dl config pipeline.
|
||||
|
||||
Covers two things:
|
||||
1. `_get_default_config()` now includes per-platform defaults under
|
||||
`extractor.<platform>` so the generated config is self-contained and
|
||||
editable via the Settings view / gallery-dl.conf.
|
||||
2. `_build_config_for_source` deep-merges user overrides: the platform
|
||||
section from `self._base_config` survives, source-specific keys
|
||||
(content_types → files/include, directory_pattern, filename_pattern,
|
||||
sleep, skip, metadata) are applied on top, and missing platform
|
||||
sections are re-seeded from defaults so behavior stays sticky.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.gallery_dl import GalleryDLService, SourceConfig
|
||||
|
||||
|
||||
def _make_service(base_config: dict = None):
|
||||
"""Build a service with an optional base config (bypasses disk IO)."""
|
||||
with patch.object(GalleryDLService, "_load_base_config", return_value=base_config or {}):
|
||||
return GalleryDLService(rate_limit=1.0)
|
||||
|
||||
|
||||
def test_default_config_includes_platform_sections():
|
||||
"""_get_default_config() bakes PLATFORM_DEFAULTS into extractor.<platform>."""
|
||||
service = _make_service()
|
||||
cfg = service._get_default_config()
|
||||
|
||||
assert "extractor" in cfg
|
||||
for platform in ("patreon", "discord", "subscribestar", "hentaifoundry", "pixiv", "deviantart"):
|
||||
assert platform in cfg["extractor"], f"{platform} missing from default config"
|
||||
|
||||
|
||||
def test_default_config_strips_content_types_meta_key():
|
||||
"""`content_types` is an internal abstraction — must not leak into the
|
||||
gallery-dl config (it gets translated to `files`/`include` per platform)."""
|
||||
service = _make_service()
|
||||
cfg = service._get_default_config()
|
||||
|
||||
for platform_section in cfg["extractor"].values():
|
||||
if isinstance(platform_section, dict):
|
||||
assert "content_types" not in platform_section
|
||||
|
||||
|
||||
def test_discord_embeds_is_not_bool_regression():
|
||||
"""Regression: `embeds: True` caused `argument of type 'bool' is not
|
||||
iterable` at runtime. Must be a string or list (the gallery-dl discord
|
||||
extractor iterates it to filter embed types)."""
|
||||
service = _make_service()
|
||||
cfg = service._get_default_config()
|
||||
|
||||
embeds = cfg["extractor"]["discord"]["embeds"]
|
||||
assert not isinstance(embeds, bool), f"discord.embeds must not be bool (got {embeds!r})"
|
||||
assert isinstance(embeds, (str, list))
|
||||
|
||||
|
||||
def test_build_config_preserves_user_overrides_on_platform_section():
|
||||
"""If the user set extractor.discord.embeds to a custom value, it must
|
||||
survive _build_config_for_source (that's the whole point of unification)."""
|
||||
user_config = {
|
||||
"extractor": {
|
||||
"discord": {"embeds": ["image"], "stickers": False},
|
||||
},
|
||||
}
|
||||
service = _make_service(user_config)
|
||||
result = service._build_config_for_source(
|
||||
platform="discord",
|
||||
source_config=SourceConfig(content_types=["all"]),
|
||||
destination="/data/downloads/Me/discord",
|
||||
)
|
||||
assert result["extractor"]["discord"]["embeds"] == ["image"]
|
||||
assert result["extractor"]["discord"]["stickers"] is False
|
||||
|
||||
|
||||
def test_build_config_reseeds_platform_when_missing_from_user_conf():
|
||||
"""If user's conf has no discord section, re-inject defaults so
|
||||
behavior stays sticky (deleting extractor.discord shouldn't silently
|
||||
break downloads)."""
|
||||
user_config = {"extractor": {"patreon": {"files": []}}} # discord missing
|
||||
service = _make_service(user_config)
|
||||
result = service._build_config_for_source(
|
||||
platform="discord",
|
||||
source_config=SourceConfig(content_types=["all"]),
|
||||
destination="/data/downloads/Me/discord",
|
||||
)
|
||||
|
||||
discord_section = result["extractor"]["discord"]
|
||||
# Re-seeded defaults: embeds, stickers, threads, directory, filename
|
||||
assert "embeds" in discord_section
|
||||
assert "stickers" in discord_section
|
||||
assert "threads" in discord_section
|
||||
|
||||
|
||||
def test_source_config_overrides_win_over_user_conf():
|
||||
"""Per-source settings (directory_pattern, filename_pattern, sleep) must
|
||||
win over user conf — source-specific is more specific than global."""
|
||||
user_config = {
|
||||
"extractor": {
|
||||
"discord": {"directory": ["global_pattern"], "filename": "{id}.{extension}"},
|
||||
},
|
||||
}
|
||||
service = _make_service(user_config)
|
||||
result = service._build_config_for_source(
|
||||
platform="discord",
|
||||
source_config=SourceConfig(
|
||||
content_types=["all"],
|
||||
directory_pattern=["source_pattern"],
|
||||
filename_pattern="{date}_{id}.{extension}",
|
||||
sleep=5.0,
|
||||
),
|
||||
destination="/data/downloads/Me/discord",
|
||||
)
|
||||
|
||||
assert result["extractor"]["discord"]["directory"] == ["source_pattern"]
|
||||
assert result["extractor"]["discord"]["filename"] == "{date}_{id}.{extension}"
|
||||
assert result["extractor"]["sleep"] == 5.0
|
||||
|
||||
|
||||
def test_build_config_preserves_user_extractor_globals():
|
||||
"""User-set extractor-level keys (e.g., retries, timeout, custom archive
|
||||
path) must survive unless overridden by source config."""
|
||||
user_config = {
|
||||
"extractor": {
|
||||
"retries": 10,
|
||||
"timeout": 60.0,
|
||||
"archive": "/custom/archive.db",
|
||||
},
|
||||
}
|
||||
service = _make_service(user_config)
|
||||
result = service._build_config_for_source(
|
||||
platform="patreon",
|
||||
source_config=SourceConfig(content_types=["all"]),
|
||||
destination="/data/downloads/Me/patreon",
|
||||
)
|
||||
|
||||
assert result["extractor"]["retries"] == 10
|
||||
assert result["extractor"]["timeout"] == 60.0
|
||||
assert result["extractor"]["archive"] == "/custom/archive.db"
|
||||
|
||||
|
||||
def test_build_config_patreon_content_types_translated_to_files():
|
||||
"""Legacy behavior preserved: patreon content_types → files list."""
|
||||
service = _make_service()
|
||||
result = service._build_config_for_source(
|
||||
platform="patreon",
|
||||
source_config=SourceConfig(content_types=["images", "attachments"]),
|
||||
destination="/data/downloads/Me/patreon",
|
||||
)
|
||||
assert result["extractor"]["patreon"]["files"] == ["images", "attachments"]
|
||||
|
||||
|
||||
def test_build_config_hentaifoundry_content_types_translated_to_include():
|
||||
"""Legacy behavior preserved: hentaifoundry 'all' content_type → include."""
|
||||
service = _make_service()
|
||||
result = service._build_config_for_source(
|
||||
platform="hentaifoundry",
|
||||
source_config=SourceConfig(content_types=["all"]),
|
||||
destination="/data/downloads/Me/hentaifoundry",
|
||||
)
|
||||
assert result["extractor"]["hentaifoundry"]["include"] == "all"
|
||||
@@ -22,6 +22,7 @@ def test_run_stats_all_zeros_on_empty_output():
|
||||
"skipped_count": 0,
|
||||
"per_item_failures": 0,
|
||||
"warning_count": 0,
|
||||
"tier_gated_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -79,3 +80,18 @@ def test_run_stats_exit_code_passed_through():
|
||||
assert service._compute_run_stats(0, "", "")["exit_code"] == 0
|
||||
assert service._compute_run_stats(4, "", "")["exit_code"] == 4
|
||||
assert service._compute_run_stats(-9, "", "")["exit_code"] == -9
|
||||
|
||||
|
||||
def test_run_stats_tier_gated_count_from_patreon_warnings():
|
||||
"""'Not allowed to view post N' warnings tallied separately from generic warnings."""
|
||||
service = _make_service()
|
||||
stderr = (
|
||||
"[patreon][warning] Not allowed to view post 100\n"
|
||||
"[patreon][warning] Not allowed to view post 101\n"
|
||||
"[patreon][warning] Not allowed to view post 102\n"
|
||||
"[downloader.http][warning] '404 OK' for 'https://example.com/x'\n"
|
||||
)
|
||||
stats = service._compute_run_stats(4, "", stderr)
|
||||
assert stats["tier_gated_count"] == 3
|
||||
# Still tallied in warning_count — tier_gated is a subset, not separate
|
||||
assert stats["warning_count"] == 4
|
||||
|
||||
Reference in New Issue
Block a user