This repository has been archived on 2026-05-31. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
GallerySubscriber/backend/tests/services/test_gallery_dl_config_merge.py
T
bvandeusen 86bf43c80a feat(downloader): tier-limited classification, config defaults, yt-dlp support
- Classify Patreon "Not allowed to view post" warnings as TIER_LIMITED so
  subscription-gated runs are distinguished from genuine failures, and persist
  error_type/message on completed runs so the distinction survives to the UI.
- Fold PLATFORM_DEFAULTS into _get_default_config so gallery-dl.conf is a
  complete, editable document; _build_config_for_source now preserves the
  user's conf and only re-seeds missing platform sections.
- Add Reset-to-Defaults action in Settings (vs. Revert Changes which just
  reloads disk); show info banner when no gallery-dl.conf exists yet.
- Fix Discord embeds option: must be "all" string, not bool (gallery-dl
  iterates the value).
- Install yt-dlp + ffmpeg in Docker images so Patreon/Mux HLS video posts
  download instead of logging "Cannot import yt-dlp" and skipping.
- Recognize yt-dlp import failures as per-item errors so they don't mask
  TIER_LIMITED/NO_NEW_CONTENT classification.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-20 19:32:44 -04:00

161 lines
6.4 KiB
Python

"""Unit tests for the unified gallery-dl config pipeline.
Covers two things:
1. `_get_default_config()` now includes per-platform defaults under
`extractor.<platform>` so the generated config is self-contained and
editable via the Settings view / gallery-dl.conf.
2. `_build_config_for_source` deep-merges user overrides: the platform
section from `self._base_config` survives, source-specific keys
(content_types → files/include, directory_pattern, filename_pattern,
sleep, skip, metadata) are applied on top, and missing platform
sections are re-seeded from defaults so behavior stays sticky.
"""
from unittest.mock import patch
from app.services.gallery_dl import GalleryDLService, SourceConfig
def _make_service(base_config: dict = None):
"""Build a service with an optional base config (bypasses disk IO)."""
with patch.object(GalleryDLService, "_load_base_config", return_value=base_config or {}):
return GalleryDLService(rate_limit=1.0)
def test_default_config_includes_platform_sections():
"""_get_default_config() bakes PLATFORM_DEFAULTS into extractor.<platform>."""
service = _make_service()
cfg = service._get_default_config()
assert "extractor" in cfg
for platform in ("patreon", "discord", "subscribestar", "hentaifoundry", "pixiv", "deviantart"):
assert platform in cfg["extractor"], f"{platform} missing from default config"
def test_default_config_strips_content_types_meta_key():
"""`content_types` is an internal abstraction — must not leak into the
gallery-dl config (it gets translated to `files`/`include` per platform)."""
service = _make_service()
cfg = service._get_default_config()
for platform_section in cfg["extractor"].values():
if isinstance(platform_section, dict):
assert "content_types" not in platform_section
def test_discord_embeds_is_not_bool_regression():
"""Regression: `embeds: True` caused `argument of type 'bool' is not
iterable` at runtime. Must be a string or list (the gallery-dl discord
extractor iterates it to filter embed types)."""
service = _make_service()
cfg = service._get_default_config()
embeds = cfg["extractor"]["discord"]["embeds"]
assert not isinstance(embeds, bool), f"discord.embeds must not be bool (got {embeds!r})"
assert isinstance(embeds, (str, list))
def test_build_config_preserves_user_overrides_on_platform_section():
"""If the user set extractor.discord.embeds to a custom value, it must
survive _build_config_for_source (that's the whole point of unification)."""
user_config = {
"extractor": {
"discord": {"embeds": ["image"], "stickers": False},
},
}
service = _make_service(user_config)
result = service._build_config_for_source(
platform="discord",
source_config=SourceConfig(content_types=["all"]),
destination="/data/downloads/Me/discord",
)
assert result["extractor"]["discord"]["embeds"] == ["image"]
assert result["extractor"]["discord"]["stickers"] is False
def test_build_config_reseeds_platform_when_missing_from_user_conf():
"""If user's conf has no discord section, re-inject defaults so
behavior stays sticky (deleting extractor.discord shouldn't silently
break downloads)."""
user_config = {"extractor": {"patreon": {"files": []}}} # discord missing
service = _make_service(user_config)
result = service._build_config_for_source(
platform="discord",
source_config=SourceConfig(content_types=["all"]),
destination="/data/downloads/Me/discord",
)
discord_section = result["extractor"]["discord"]
# Re-seeded defaults: embeds, stickers, threads, directory, filename
assert "embeds" in discord_section
assert "stickers" in discord_section
assert "threads" in discord_section
def test_source_config_overrides_win_over_user_conf():
"""Per-source settings (directory_pattern, filename_pattern, sleep) must
win over user conf — source-specific is more specific than global."""
user_config = {
"extractor": {
"discord": {"directory": ["global_pattern"], "filename": "{id}.{extension}"},
},
}
service = _make_service(user_config)
result = service._build_config_for_source(
platform="discord",
source_config=SourceConfig(
content_types=["all"],
directory_pattern=["source_pattern"],
filename_pattern="{date}_{id}.{extension}",
sleep=5.0,
),
destination="/data/downloads/Me/discord",
)
assert result["extractor"]["discord"]["directory"] == ["source_pattern"]
assert result["extractor"]["discord"]["filename"] == "{date}_{id}.{extension}"
assert result["extractor"]["sleep"] == 5.0
def test_build_config_preserves_user_extractor_globals():
"""User-set extractor-level keys (e.g., retries, timeout, custom archive
path) must survive unless overridden by source config."""
user_config = {
"extractor": {
"retries": 10,
"timeout": 60.0,
"archive": "/custom/archive.db",
},
}
service = _make_service(user_config)
result = service._build_config_for_source(
platform="patreon",
source_config=SourceConfig(content_types=["all"]),
destination="/data/downloads/Me/patreon",
)
assert result["extractor"]["retries"] == 10
assert result["extractor"]["timeout"] == 60.0
assert result["extractor"]["archive"] == "/custom/archive.db"
def test_build_config_patreon_content_types_translated_to_files():
"""Legacy behavior preserved: patreon content_types → files list."""
service = _make_service()
result = service._build_config_for_source(
platform="patreon",
source_config=SourceConfig(content_types=["images", "attachments"]),
destination="/data/downloads/Me/patreon",
)
assert result["extractor"]["patreon"]["files"] == ["images", "attachments"]
def test_build_config_hentaifoundry_content_types_translated_to_include():
"""Legacy behavior preserved: hentaifoundry 'all' content_type → include."""
service = _make_service()
result = service._build_config_for_source(
platform="hentaifoundry",
source_config=SourceConfig(content_types=["all"]),
destination="/data/downloads/Me/hentaifoundry",
)
assert result["extractor"]["hentaifoundry"]["include"] == "all"