feat(downloader): tier-limited classification, config defaults, yt-dlp support
- Classify Patreon "Not allowed to view post" warnings as TIER_LIMITED so subscription-gated runs are distinguished from genuine failures, and persist error_type/message on completed runs so the distinction survives to the UI. - Fold PLATFORM_DEFAULTS into _get_default_config so gallery-dl.conf is a complete, editable document; _build_config_for_source now preserves the user's conf and only re-seeds missing platform sections. - Add Reset-to-Defaults action in Settings (vs. Revert Changes which just reloads disk); show info banner when no gallery-dl.conf exists yet. - Fix Discord embeds option: must be "all" string, not bool (gallery-dl iterates the value). - Install yt-dlp + ffmpeg in Docker images so Patreon/Mux HLS video posts download instead of logging "Cannot import yt-dlp" and skipping. - Recognize yt-dlp import failures as per-item errors so they don't mask TIER_LIMITED/NO_NEW_CONTENT classification. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
|
|||||||
gcc \
|
gcc \
|
||||||
libpq-dev \
|
libpq-dev \
|
||||||
gosu \
|
gosu \
|
||||||
|
ffmpeg \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Install Python dependencies
|
# Install Python dependencies
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ WORKDIR /app
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
gcc \
|
gcc \
|
||||||
libpq-dev \
|
libpq-dev \
|
||||||
|
ffmpeg \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Install Python dependencies
|
# Install Python dependencies
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ class ErrorType(str, Enum):
|
|||||||
ACCESS_DENIED = "access_denied"
|
ACCESS_DENIED = "access_denied"
|
||||||
NETWORK_ERROR = "network_error"
|
NETWORK_ERROR = "network_error"
|
||||||
NO_NEW_CONTENT = "no_new_content"
|
NO_NEW_CONTENT = "no_new_content"
|
||||||
|
TIER_LIMITED = "tier_limited"
|
||||||
TIMEOUT = "timeout"
|
TIMEOUT = "timeout"
|
||||||
HTTP_ERROR = "http_error"
|
HTTP_ERROR = "http_error"
|
||||||
UNSUPPORTED_URL = "unsupported_url"
|
UNSUPPORTED_URL = "unsupported_url"
|
||||||
@@ -229,8 +230,10 @@ class GalleryDLService:
|
|||||||
"content_types": ["all"],
|
"content_types": ["all"],
|
||||||
"directory": ["{channel[name]}"],
|
"directory": ["{channel[name]}"],
|
||||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||||||
# Additional options
|
# Additional options. gallery-dl's discord extractor treats `embeds`
|
||||||
"embeds": True, # Download embedded URLs
|
# as an iterable of embed types (strings) or the string "all" — a
|
||||||
|
# bool causes "argument of type 'bool' is not iterable" at runtime.
|
||||||
|
"embeds": "all",
|
||||||
"stickers": True, # Download stickers
|
"stickers": True, # Download stickers
|
||||||
"reactions": False, # Skip reaction images
|
"reactions": False, # Skip reaction images
|
||||||
# Deep crawling options
|
# Deep crawling options
|
||||||
@@ -281,8 +284,16 @@ class GalleryDLService:
|
|||||||
return self._get_default_config()
|
return self._get_default_config()
|
||||||
|
|
||||||
def _get_default_config(self) -> dict:
|
def _get_default_config(self) -> dict:
|
||||||
"""Get default gallery-dl configuration."""
|
"""Get default gallery-dl configuration.
|
||||||
return {
|
|
||||||
|
Platform-specific defaults from PLATFORM_DEFAULTS are folded into
|
||||||
|
extractor.<platform> so the returned dict is a complete, self-contained
|
||||||
|
gallery-dl config — editable via the Settings view and writable to
|
||||||
|
gallery-dl.conf for full user control. The internal `content_types`
|
||||||
|
meta-key is stripped (it's our abstraction, not a gallery-dl option —
|
||||||
|
it gets translated per-platform to `files`/`include` at run time).
|
||||||
|
"""
|
||||||
|
config = {
|
||||||
"extractor": {
|
"extractor": {
|
||||||
"base-directory": self.settings.download_path,
|
"base-directory": self.settings.download_path,
|
||||||
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
|
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
|
||||||
@@ -312,6 +323,11 @@ class GalleryDLService:
|
|||||||
"progress": True,
|
"progress": True,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
|
||||||
|
config["extractor"][platform_name] = {
|
||||||
|
key: value for key, value in defaults.items() if key != "content_types"
|
||||||
|
}
|
||||||
|
return config
|
||||||
|
|
||||||
def _build_config_for_source(
|
def _build_config_for_source(
|
||||||
self,
|
self,
|
||||||
@@ -321,30 +337,33 @@ class GalleryDLService:
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""Build a merged configuration for a specific source.
|
"""Build a merged configuration for a specific source.
|
||||||
|
|
||||||
Merges: base config → platform defaults → source overrides
|
Precedence: _get_default_config() (built-in) → gallery-dl.conf on disk
|
||||||
|
→ per-source overrides. The platform's extractor section is preserved
|
||||||
|
from the base config (deep copy) and source overrides are applied on
|
||||||
|
top — so user tweaks in gallery-dl.conf survive. If the user's conf
|
||||||
|
dropped the platform section entirely, re-seed from defaults so
|
||||||
|
platform behavior stays sticky.
|
||||||
"""
|
"""
|
||||||
config = json.loads(json.dumps(self._base_config)) # Deep copy
|
config = json.loads(json.dumps(self._base_config)) # Deep copy
|
||||||
|
|
||||||
# Get platform defaults
|
|
||||||
platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {})
|
|
||||||
|
|
||||||
# Ensure extractor section exists
|
|
||||||
if "extractor" not in config:
|
if "extractor" not in config:
|
||||||
config["extractor"] = {}
|
config["extractor"] = {}
|
||||||
|
|
||||||
# Apply base directory
|
# Re-seed platform defaults if the user's conf is missing the section.
|
||||||
config["extractor"]["base-directory"] = destination
|
# Prevents "I deleted extractor.discord from my conf and silently lost
|
||||||
|
# embeds/stickers/threads." from becoming a footgun.
|
||||||
|
if platform not in config["extractor"]:
|
||||||
|
seed = self._get_default_config()
|
||||||
|
config["extractor"][platform] = seed.get("extractor", {}).get(platform, {})
|
||||||
|
|
||||||
# Apply rate limiting overrides
|
# extractor.* global overrides from source config
|
||||||
|
config["extractor"]["base-directory"] = destination
|
||||||
if source_config.sleep is not None:
|
if source_config.sleep is not None:
|
||||||
config["extractor"]["sleep"] = source_config.sleep
|
config["extractor"]["sleep"] = source_config.sleep
|
||||||
if source_config.sleep_request is not None:
|
if source_config.sleep_request is not None:
|
||||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||||
|
|
||||||
# Apply skip setting
|
|
||||||
config["extractor"]["skip"] = source_config.skip_existing
|
config["extractor"]["skip"] = source_config.skip_existing
|
||||||
|
|
||||||
# Configure metadata postprocessor
|
|
||||||
if source_config.save_metadata:
|
if source_config.save_metadata:
|
||||||
config["extractor"]["postprocessors"] = [
|
config["extractor"]["postprocessors"] = [
|
||||||
{
|
{
|
||||||
@@ -357,43 +376,29 @@ class GalleryDLService:
|
|||||||
else:
|
else:
|
||||||
config["extractor"].pop("postprocessors", None)
|
config["extractor"].pop("postprocessors", None)
|
||||||
|
|
||||||
# Build platform-specific section
|
# Source overrides on the platform-specific section. We preserve
|
||||||
platform_config = {}
|
# whatever is in the base config (platform defaults or user's conf)
|
||||||
|
# and only set keys that the source explicitly controls.
|
||||||
|
platform_section = config["extractor"][platform]
|
||||||
|
|
||||||
# Apply all platform defaults first (videos, embeds, stickers, etc.)
|
# Content types translate to platform-specific gallery-dl keys.
|
||||||
for key, value in platform_defaults.items():
|
|
||||||
if key not in ["content_types", "directory", "filename"]:
|
|
||||||
platform_config[key] = value
|
|
||||||
|
|
||||||
# Content types (platform-specific handling)
|
|
||||||
if platform == "patreon":
|
if platform == "patreon":
|
||||||
# Patreon uses "files" array for content types
|
|
||||||
if "all" in source_config.content_types:
|
if "all" in source_config.content_types:
|
||||||
platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"]
|
platform_section["files"] = ["images", "image_large", "attachments", "postfile", "content"]
|
||||||
else:
|
else:
|
||||||
platform_config["files"] = source_config.content_types
|
platform_section["files"] = source_config.content_types
|
||||||
elif platform == "hentaifoundry":
|
elif platform == "hentaifoundry":
|
||||||
# HentaiFoundry uses "include" for content filtering
|
|
||||||
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
||||||
platform_config["include"] = "all"
|
platform_section["include"] = "all"
|
||||||
|
|
||||||
# Directory pattern
|
# Directory/filename: source override wins; otherwise inherit whatever
|
||||||
|
# the base config already has (platform default or user-set value).
|
||||||
if source_config.directory_pattern:
|
if source_config.directory_pattern:
|
||||||
platform_config["directory"] = source_config.directory_pattern
|
platform_section["directory"] = source_config.directory_pattern
|
||||||
elif "directory" in platform_defaults:
|
|
||||||
platform_config["directory"] = platform_defaults["directory"]
|
|
||||||
|
|
||||||
# Filename pattern
|
|
||||||
if source_config.filename_pattern:
|
if source_config.filename_pattern:
|
||||||
platform_config["filename"] = source_config.filename_pattern
|
platform_section["filename"] = source_config.filename_pattern
|
||||||
elif "filename" in platform_defaults:
|
|
||||||
platform_config["filename"] = platform_defaults["filename"]
|
|
||||||
|
|
||||||
# Always save metadata at platform level too
|
platform_section["metadata"] = source_config.save_metadata
|
||||||
platform_config["metadata"] = source_config.save_metadata
|
|
||||||
|
|
||||||
# Add platform config to extractor
|
|
||||||
config["extractor"][platform] = platform_config
|
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
@@ -450,6 +455,12 @@ class GalleryDLService:
|
|||||||
"unable to get post",
|
"unable to get post",
|
||||||
"failed to extract campaign id",
|
"failed to extract campaign id",
|
||||||
"failed to download",
|
"failed to download",
|
||||||
|
# yt-dlp/youtube-dl missing: each video that gallery-dl tries to
|
||||||
|
# download fails, but image/skip activity continues. Treat as
|
||||||
|
# per-item so it doesn't mask TIER_LIMITED/NO_NEW_CONTENT.
|
||||||
|
# Surface separately as a setup gap (see Dockerfile yt-dlp install).
|
||||||
|
"cannot import yt-dlp",
|
||||||
|
"cannot import youtube-dl",
|
||||||
]
|
]
|
||||||
error_lines = [
|
error_lines = [
|
||||||
line for line in combined.split('\n')
|
line for line in combined.split('\n')
|
||||||
@@ -547,6 +558,23 @@ class GalleryDLService:
|
|||||||
logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
|
logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
|
||||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||||
|
|
||||||
|
# Subscription-tier gating: gallery-dl emits [patreon][warning] "Not allowed to
|
||||||
|
# view post N" for posts the current auth token can't access. A run that exits
|
||||||
|
# 1/4 with only these warnings and no downloads means the account's tier doesn't
|
||||||
|
# cover the requested posts — it's a legitimate state, not a failure.
|
||||||
|
if return_code in (1, 4) and not has_actual_error:
|
||||||
|
tier_gated_lines = [
|
||||||
|
line for line in combined.split('\n')
|
||||||
|
if "][warning]" in line and "not allowed to view post" in line
|
||||||
|
]
|
||||||
|
if tier_gated_lines:
|
||||||
|
count = len(tier_gated_lines)
|
||||||
|
logger.debug(f"Error classified as tier_limited via: rc{return_code} + {count} tier-gated warnings")
|
||||||
|
return (
|
||||||
|
ErrorType.TIER_LIMITED,
|
||||||
|
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||||
|
)
|
||||||
|
|
||||||
# Final fallback - genuine unknown error
|
# Final fallback - genuine unknown error
|
||||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||||
|
|
||||||
@@ -573,6 +601,8 @@ class GalleryDLService:
|
|||||||
- per_item_failures: [download][error] lines (single-item failures the
|
- per_item_failures: [download][error] lines (single-item failures the
|
||||||
downloader logs but recovers from)
|
downloader logs but recovers from)
|
||||||
- warning_count: [*][warning] lines
|
- warning_count: [*][warning] lines
|
||||||
|
- tier_gated_count: 'not allowed to view post' warnings (Patreon
|
||||||
|
subscription-tier gating — informational, not a failure)
|
||||||
"""
|
"""
|
||||||
stdout = stdout or ""
|
stdout = stdout or ""
|
||||||
stderr = stderr or ""
|
stderr = stderr or ""
|
||||||
@@ -593,12 +623,18 @@ class GalleryDLService:
|
|||||||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tier_gated_count = sum(
|
||||||
|
1 for line in stderr.splitlines()
|
||||||
|
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"exit_code": return_code,
|
"exit_code": return_code,
|
||||||
"downloaded_count": self._count_downloaded_files(stdout),
|
"downloaded_count": self._count_downloaded_files(stdout),
|
||||||
"skipped_count": skipped_stdout + skipped_stderr,
|
"skipped_count": skipped_stdout + skipped_stderr,
|
||||||
"per_item_failures": per_item_failures,
|
"per_item_failures": per_item_failures,
|
||||||
"warning_count": warning_count,
|
"warning_count": warning_count,
|
||||||
|
"tier_gated_count": tier_gated_count,
|
||||||
}
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -769,8 +805,10 @@ class GalleryDLService:
|
|||||||
proc.returncode, proc.stdout, proc.stderr
|
proc.returncode, proc.stdout, proc.stderr
|
||||||
)
|
)
|
||||||
|
|
||||||
# NO_NEW_CONTENT is considered successful
|
# NO_NEW_CONTENT and TIER_LIMITED are non-failure states — the run completed
|
||||||
success = error_type == ErrorType.NO_NEW_CONTENT
|
# as well as it could given external constraints (archive up-to-date, or
|
||||||
|
# current subscription tier can't see the requested posts).
|
||||||
|
success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
|
||||||
|
|
||||||
return DownloadResult(
|
return DownloadResult(
|
||||||
success=success,
|
success=success,
|
||||||
|
|||||||
@@ -420,6 +420,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
|||||||
|
|
||||||
if dl_result.success:
|
if dl_result.success:
|
||||||
download.status = DownloadStatus.COMPLETED
|
download.status = DownloadStatus.COMPLETED
|
||||||
|
# Some success states still carry an informational error_type
|
||||||
|
# (e.g., TIER_LIMITED — run completed, but the subscription tier
|
||||||
|
# blocked posts we'd have otherwise downloaded). Persist it so
|
||||||
|
# the UI can distinguish "fully succeeded" from "succeeded with
|
||||||
|
# external constraints".
|
||||||
|
download.error_type = dl_result.error_type.value if dl_result.error_type else None
|
||||||
|
download.error_message = dl_result.error_message
|
||||||
source.last_success = utcnow()
|
source.last_success = utcnow()
|
||||||
source.error_count = 0
|
source.error_count = 0
|
||||||
logger.info(f"Download completed for {subscription_name}/{source_platform}: {dl_result.files_downloaded} files")
|
logger.info(f"Download completed for {subscription_name}/{source_platform}: {dl_result.files_downloaded} files")
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ pyyaml
|
|||||||
# gallery-dl (>=1.31.10 fixes Patreon HTTP 426 / OAuth flow)
|
# gallery-dl (>=1.31.10 fixes Patreon HTTP 426 / OAuth flow)
|
||||||
gallery-dl>=1.31.10
|
gallery-dl>=1.31.10
|
||||||
|
|
||||||
|
# yt-dlp: video backend gallery-dl shells out to for Patreon/Mux HLS streams,
|
||||||
|
# embedded YouTube, etc. Without it, video posts log "[downloader.ytdl][error]
|
||||||
|
# Cannot import yt-dlp" and skip. Requires ffmpeg in the system image to mux
|
||||||
|
# HLS segments into mp4.
|
||||||
|
yt-dlp
|
||||||
|
|
||||||
# Testing
|
# Testing
|
||||||
pytest
|
pytest
|
||||||
pytest-asyncio
|
pytest-asyncio
|
||||||
|
|||||||
@@ -93,3 +93,94 @@ def test_patreon_per_item_download_failure_without_skips_still_fails():
|
|||||||
error_type, _message = service._categorize_error(1, stdout, stderr)
|
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||||
|
|
||||||
assert error_type != ErrorType.NO_NEW_CONTENT
|
assert error_type != ErrorType.NO_NEW_CONTENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_patreon_tier_gated_warnings_classify_as_tier_limited():
|
||||||
|
"""Many 'Not allowed to view post N' warnings + exit 4 + no downloads →
|
||||||
|
TIER_LIMITED. Reflects a Patreon tier downgrade where older posts are now
|
||||||
|
paywalled but we don't want to call this a failure."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = "\n".join(
|
||||||
|
f"[patreon][warning] Not allowed to view post {100000 + i}"
|
||||||
|
for i in range(15)
|
||||||
|
)
|
||||||
|
stdout = ""
|
||||||
|
|
||||||
|
error_type, message = service._categorize_error(4, stdout, stderr)
|
||||||
|
|
||||||
|
assert error_type == ErrorType.TIER_LIMITED
|
||||||
|
assert "15" in message
|
||||||
|
|
||||||
|
|
||||||
|
def test_tier_limited_with_exit_code_1_also_classifies():
|
||||||
|
"""Mixed run: exit 1 with tier-gated warnings and no real errors."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||||
|
stdout = ""
|
||||||
|
|
||||||
|
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||||
|
|
||||||
|
assert error_type == ErrorType.TIER_LIMITED
|
||||||
|
|
||||||
|
|
||||||
|
def test_tier_limited_not_applied_when_real_error_present():
|
||||||
|
"""Tier warnings + a real auth error → auth error wins (TIER_LIMITED is
|
||||||
|
only for 'everything else was fine' runs)."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = (
|
||||||
|
"[patreon][warning] Not allowed to view post 999\n"
|
||||||
|
'[patreon][error] "GET /api/posts HTTP/1.1" 401 None\n'
|
||||||
|
)
|
||||||
|
stdout = ""
|
||||||
|
|
||||||
|
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||||
|
|
||||||
|
assert error_type == ErrorType.AUTH_ERROR
|
||||||
|
|
||||||
|
|
||||||
|
def test_tier_limited_does_not_fire_on_exit_0():
|
||||||
|
"""Successful run with some paywalled posts as warnings isn't an error at
|
||||||
|
all — don't reach the TIER_LIMITED branch (exit 0 → success path earlier)."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||||
|
stdout = "/data/downloads/Artist/patreon/post1/img.jpg\n"
|
||||||
|
|
||||||
|
error_type, _message = service._categorize_error(0, stdout, stderr)
|
||||||
|
|
||||||
|
# Exit 0 + stdout activity should not hit the TIER_LIMITED branch
|
||||||
|
assert error_type != ErrorType.TIER_LIMITED
|
||||||
|
|
||||||
|
|
||||||
|
def test_skip_activity_takes_precedence_over_tier_limited():
|
||||||
|
"""If there's also skip activity, NO_NEW_CONTENT wins — precedence matters
|
||||||
|
because the tier_limited branch is the last informational fallback."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = "[patreon][warning] Not allowed to view post 999\n"
|
||||||
|
stdout = "# /data/downloads/Artist/patreon/post1/img.jpg\n"
|
||||||
|
|
||||||
|
error_type, _message = service._categorize_error(1, stdout, stderr)
|
||||||
|
|
||||||
|
assert error_type == ErrorType.NO_NEW_CONTENT
|
||||||
|
|
||||||
|
|
||||||
|
def test_ytdl_missing_with_skips_classifies_as_no_new_content():
|
||||||
|
"""Real-world scenario from a Patreon source: container is missing yt-dlp
|
||||||
|
so every video logs '[downloader.ytdl][error] Cannot import yt-dlp' plus
|
||||||
|
one [download][error] per video. With overwhelming skip activity, this
|
||||||
|
should classify as NO_NEW_CONTENT (the setup gap is per-item — it
|
||||||
|
shouldn't mask the dominant signal that the archive is up-to-date).
|
||||||
|
|
||||||
|
Regression: previously the yt-dlp import error didn't match any per-item
|
||||||
|
pattern, so has_actual_error stayed True and the run was misclassified as
|
||||||
|
UNKNOWN_ERROR even though 1300+ files were skipped."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = (
|
||||||
|
"[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl\n"
|
||||||
|
"[download][error] Failed to download 02_video.mp4\n"
|
||||||
|
"[download][error] Failed to download 02_video.mp4\n"
|
||||||
|
)
|
||||||
|
stdout = "# /data/downloads/Artist/patreon/post/01_img.jpg\n" * 100
|
||||||
|
|
||||||
|
error_type, _message = service._categorize_error(4, stdout, stderr)
|
||||||
|
|
||||||
|
assert error_type == ErrorType.NO_NEW_CONTENT
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
"""Unit tests for the unified gallery-dl config pipeline.
|
||||||
|
|
||||||
|
Covers two things:
|
||||||
|
1. `_get_default_config()` now includes per-platform defaults under
|
||||||
|
`extractor.<platform>` so the generated config is self-contained and
|
||||||
|
editable via the Settings view / gallery-dl.conf.
|
||||||
|
2. `_build_config_for_source` deep-merges user overrides: the platform
|
||||||
|
section from `self._base_config` survives, source-specific keys
|
||||||
|
(content_types → files/include, directory_pattern, filename_pattern,
|
||||||
|
sleep, skip, metadata) are applied on top, and missing platform
|
||||||
|
sections are re-seeded from defaults so behavior stays sticky.
|
||||||
|
"""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from app.services.gallery_dl import GalleryDLService, SourceConfig
|
||||||
|
|
||||||
|
|
||||||
|
def _make_service(base_config: dict = None):
|
||||||
|
"""Build a service with an optional base config (bypasses disk IO)."""
|
||||||
|
with patch.object(GalleryDLService, "_load_base_config", return_value=base_config or {}):
|
||||||
|
return GalleryDLService(rate_limit=1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_config_includes_platform_sections():
|
||||||
|
"""_get_default_config() bakes PLATFORM_DEFAULTS into extractor.<platform>."""
|
||||||
|
service = _make_service()
|
||||||
|
cfg = service._get_default_config()
|
||||||
|
|
||||||
|
assert "extractor" in cfg
|
||||||
|
for platform in ("patreon", "discord", "subscribestar", "hentaifoundry", "pixiv", "deviantart"):
|
||||||
|
assert platform in cfg["extractor"], f"{platform} missing from default config"
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_config_strips_content_types_meta_key():
|
||||||
|
"""`content_types` is an internal abstraction — must not leak into the
|
||||||
|
gallery-dl config (it gets translated to `files`/`include` per platform)."""
|
||||||
|
service = _make_service()
|
||||||
|
cfg = service._get_default_config()
|
||||||
|
|
||||||
|
for platform_section in cfg["extractor"].values():
|
||||||
|
if isinstance(platform_section, dict):
|
||||||
|
assert "content_types" not in platform_section
|
||||||
|
|
||||||
|
|
||||||
|
def test_discord_embeds_is_not_bool_regression():
|
||||||
|
"""Regression: `embeds: True` caused `argument of type 'bool' is not
|
||||||
|
iterable` at runtime. Must be a string or list (the gallery-dl discord
|
||||||
|
extractor iterates it to filter embed types)."""
|
||||||
|
service = _make_service()
|
||||||
|
cfg = service._get_default_config()
|
||||||
|
|
||||||
|
embeds = cfg["extractor"]["discord"]["embeds"]
|
||||||
|
assert not isinstance(embeds, bool), f"discord.embeds must not be bool (got {embeds!r})"
|
||||||
|
assert isinstance(embeds, (str, list))
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_preserves_user_overrides_on_platform_section():
|
||||||
|
"""If the user set extractor.discord.embeds to a custom value, it must
|
||||||
|
survive _build_config_for_source (that's the whole point of unification)."""
|
||||||
|
user_config = {
|
||||||
|
"extractor": {
|
||||||
|
"discord": {"embeds": ["image"], "stickers": False},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
service = _make_service(user_config)
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="discord",
|
||||||
|
source_config=SourceConfig(content_types=["all"]),
|
||||||
|
destination="/data/downloads/Me/discord",
|
||||||
|
)
|
||||||
|
assert result["extractor"]["discord"]["embeds"] == ["image"]
|
||||||
|
assert result["extractor"]["discord"]["stickers"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_reseeds_platform_when_missing_from_user_conf():
|
||||||
|
"""If user's conf has no discord section, re-inject defaults so
|
||||||
|
behavior stays sticky (deleting extractor.discord shouldn't silently
|
||||||
|
break downloads)."""
|
||||||
|
user_config = {"extractor": {"patreon": {"files": []}}} # discord missing
|
||||||
|
service = _make_service(user_config)
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="discord",
|
||||||
|
source_config=SourceConfig(content_types=["all"]),
|
||||||
|
destination="/data/downloads/Me/discord",
|
||||||
|
)
|
||||||
|
|
||||||
|
discord_section = result["extractor"]["discord"]
|
||||||
|
# Re-seeded defaults: embeds, stickers, threads, directory, filename
|
||||||
|
assert "embeds" in discord_section
|
||||||
|
assert "stickers" in discord_section
|
||||||
|
assert "threads" in discord_section
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_config_overrides_win_over_user_conf():
|
||||||
|
"""Per-source settings (directory_pattern, filename_pattern, sleep) must
|
||||||
|
win over user conf — source-specific is more specific than global."""
|
||||||
|
user_config = {
|
||||||
|
"extractor": {
|
||||||
|
"discord": {"directory": ["global_pattern"], "filename": "{id}.{extension}"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
service = _make_service(user_config)
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="discord",
|
||||||
|
source_config=SourceConfig(
|
||||||
|
content_types=["all"],
|
||||||
|
directory_pattern=["source_pattern"],
|
||||||
|
filename_pattern="{date}_{id}.{extension}",
|
||||||
|
sleep=5.0,
|
||||||
|
),
|
||||||
|
destination="/data/downloads/Me/discord",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["extractor"]["discord"]["directory"] == ["source_pattern"]
|
||||||
|
assert result["extractor"]["discord"]["filename"] == "{date}_{id}.{extension}"
|
||||||
|
assert result["extractor"]["sleep"] == 5.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_preserves_user_extractor_globals():
|
||||||
|
"""User-set extractor-level keys (e.g., retries, timeout, custom archive
|
||||||
|
path) must survive unless overridden by source config."""
|
||||||
|
user_config = {
|
||||||
|
"extractor": {
|
||||||
|
"retries": 10,
|
||||||
|
"timeout": 60.0,
|
||||||
|
"archive": "/custom/archive.db",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
service = _make_service(user_config)
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="patreon",
|
||||||
|
source_config=SourceConfig(content_types=["all"]),
|
||||||
|
destination="/data/downloads/Me/patreon",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["extractor"]["retries"] == 10
|
||||||
|
assert result["extractor"]["timeout"] == 60.0
|
||||||
|
assert result["extractor"]["archive"] == "/custom/archive.db"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_patreon_content_types_translated_to_files():
|
||||||
|
"""Legacy behavior preserved: patreon content_types → files list."""
|
||||||
|
service = _make_service()
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="patreon",
|
||||||
|
source_config=SourceConfig(content_types=["images", "attachments"]),
|
||||||
|
destination="/data/downloads/Me/patreon",
|
||||||
|
)
|
||||||
|
assert result["extractor"]["patreon"]["files"] == ["images", "attachments"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_hentaifoundry_content_types_translated_to_include():
|
||||||
|
"""Legacy behavior preserved: hentaifoundry 'all' content_type → include."""
|
||||||
|
service = _make_service()
|
||||||
|
result = service._build_config_for_source(
|
||||||
|
platform="hentaifoundry",
|
||||||
|
source_config=SourceConfig(content_types=["all"]),
|
||||||
|
destination="/data/downloads/Me/hentaifoundry",
|
||||||
|
)
|
||||||
|
assert result["extractor"]["hentaifoundry"]["include"] == "all"
|
||||||
@@ -22,6 +22,7 @@ def test_run_stats_all_zeros_on_empty_output():
|
|||||||
"skipped_count": 0,
|
"skipped_count": 0,
|
||||||
"per_item_failures": 0,
|
"per_item_failures": 0,
|
||||||
"warning_count": 0,
|
"warning_count": 0,
|
||||||
|
"tier_gated_count": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -79,3 +80,18 @@ def test_run_stats_exit_code_passed_through():
|
|||||||
assert service._compute_run_stats(0, "", "")["exit_code"] == 0
|
assert service._compute_run_stats(0, "", "")["exit_code"] == 0
|
||||||
assert service._compute_run_stats(4, "", "")["exit_code"] == 4
|
assert service._compute_run_stats(4, "", "")["exit_code"] == 4
|
||||||
assert service._compute_run_stats(-9, "", "")["exit_code"] == -9
|
assert service._compute_run_stats(-9, "", "")["exit_code"] == -9
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_stats_tier_gated_count_from_patreon_warnings():
|
||||||
|
"""'Not allowed to view post N' warnings tallied separately from generic warnings."""
|
||||||
|
service = _make_service()
|
||||||
|
stderr = (
|
||||||
|
"[patreon][warning] Not allowed to view post 100\n"
|
||||||
|
"[patreon][warning] Not allowed to view post 101\n"
|
||||||
|
"[patreon][warning] Not allowed to view post 102\n"
|
||||||
|
"[downloader.http][warning] '404 OK' for 'https://example.com/x'\n"
|
||||||
|
)
|
||||||
|
stats = service._compute_run_stats(4, "", stderr)
|
||||||
|
assert stats["tier_gated_count"] == 3
|
||||||
|
# Still tallied in warning_count — tier_gated is a subset, not separate
|
||||||
|
assert stats["warning_count"] == 4
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ export const useSettingsStore = defineStore('settings', () => {
|
|||||||
async function fetchGalleryDLConfig() {
|
async function fetchGalleryDLConfig() {
|
||||||
const response = await settingsApi.getGalleryDL()
|
const response = await settingsApi.getGalleryDL()
|
||||||
galleryDLConfig.value = response.data.config
|
galleryDLConfig.value = response.data.config
|
||||||
return response.data.config
|
return response.data
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updateGalleryDLConfig(config) {
|
async function updateGalleryDLConfig(config) {
|
||||||
@@ -36,6 +36,12 @@ export const useSettingsStore = defineStore('settings', () => {
|
|||||||
return response.data.config
|
return response.data.config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function resetGalleryDLConfig() {
|
||||||
|
const response = await settingsApi.resetGalleryDL()
|
||||||
|
galleryDLConfig.value = response.data.config
|
||||||
|
return response.data.config
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
settings,
|
settings,
|
||||||
galleryDLConfig,
|
galleryDLConfig,
|
||||||
@@ -44,5 +50,6 @@ export const useSettingsStore = defineStore('settings', () => {
|
|||||||
updateSettings,
|
updateSettings,
|
||||||
fetchGalleryDLConfig,
|
fetchGalleryDLConfig,
|
||||||
updateGalleryDLConfig,
|
updateGalleryDLConfig,
|
||||||
|
resetGalleryDLConfig,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -184,6 +184,16 @@
|
|||||||
Invalid configuration may cause downloads to fail.
|
Invalid configuration may cause downloads to fail.
|
||||||
</v-alert>
|
</v-alert>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="configIsDefault"
|
||||||
|
type="info"
|
||||||
|
variant="tonal"
|
||||||
|
class="mb-4"
|
||||||
|
density="compact"
|
||||||
|
>
|
||||||
|
No gallery-dl.conf found on disk. Showing computed defaults — click Save Configuration to persist.
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
<v-textarea
|
<v-textarea
|
||||||
v-model="galleryDLConfigText"
|
v-model="galleryDLConfigText"
|
||||||
label="gallery-dl.conf (JSON)"
|
label="gallery-dl.conf (JSON)"
|
||||||
@@ -205,12 +215,20 @@
|
|||||||
Documentation
|
Documentation
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
variant="text"
|
||||||
|
color="warning"
|
||||||
|
@click="resetConfigDialog = true"
|
||||||
|
:disabled="savingConfig"
|
||||||
|
>
|
||||||
|
Reset to Defaults
|
||||||
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
variant="text"
|
variant="text"
|
||||||
@click="loadGalleryDLConfig"
|
@click="loadGalleryDLConfig"
|
||||||
:disabled="savingConfig"
|
:disabled="savingConfig"
|
||||||
>
|
>
|
||||||
Reset
|
Revert Changes
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
color="primary"
|
color="primary"
|
||||||
@@ -223,6 +241,27 @@
|
|||||||
</v-card-actions>
|
</v-card-actions>
|
||||||
</v-card>
|
</v-card>
|
||||||
|
|
||||||
|
<v-dialog v-model="resetConfigDialog" max-width="500">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Reset to Defaults?</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
This overwrites gallery-dl.conf with computed defaults, replacing any
|
||||||
|
customizations. Use this to recover after misconfiguring the file.
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="resetConfigDialog = false">Cancel</v-btn>
|
||||||
|
<v-btn
|
||||||
|
color="warning"
|
||||||
|
:loading="resettingConfig"
|
||||||
|
@click="resetGalleryDLConfigToDefaults"
|
||||||
|
>
|
||||||
|
Reset
|
||||||
|
</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
|
||||||
<!-- Platform Default Settings -->
|
<!-- Platform Default Settings -->
|
||||||
<v-card class="mb-6">
|
<v-card class="mb-6">
|
||||||
<v-card-title>
|
<v-card-title>
|
||||||
@@ -414,6 +453,9 @@ const savingConfig = ref(false)
|
|||||||
const settings = ref({})
|
const settings = ref({})
|
||||||
const galleryDLConfigText = ref('')
|
const galleryDLConfigText = ref('')
|
||||||
const configError = ref('')
|
const configError = ref('')
|
||||||
|
const configIsDefault = ref(false)
|
||||||
|
const resetConfigDialog = ref(false)
|
||||||
|
const resettingConfig = ref(false)
|
||||||
const platforms = ref({})
|
const platforms = ref({})
|
||||||
const apiHealthy = ref(true)
|
const apiHealthy = ref(true)
|
||||||
const showApiKey = ref(false)
|
const showApiKey = ref(false)
|
||||||
@@ -492,8 +534,9 @@ async function saveSettings() {
|
|||||||
|
|
||||||
async function loadGalleryDLConfig() {
|
async function loadGalleryDLConfig() {
|
||||||
try {
|
try {
|
||||||
const config = await settingsStore.fetchGalleryDLConfig()
|
const data = await settingsStore.fetchGalleryDLConfig()
|
||||||
galleryDLConfigText.value = JSON.stringify(config, null, 2)
|
galleryDLConfigText.value = JSON.stringify(data.config, null, 2)
|
||||||
|
configIsDefault.value = data.is_default === true
|
||||||
configError.value = ''
|
configError.value = ''
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
notifications.error(`Failed to load config: ${error.message}`)
|
notifications.error(`Failed to load config: ${error.message}`)
|
||||||
@@ -516,6 +559,7 @@ async function saveGalleryDLConfig() {
|
|||||||
try {
|
try {
|
||||||
const config = JSON.parse(galleryDLConfigText.value)
|
const config = JSON.parse(galleryDLConfigText.value)
|
||||||
await settingsStore.updateGalleryDLConfig(config)
|
await settingsStore.updateGalleryDLConfig(config)
|
||||||
|
configIsDefault.value = false
|
||||||
notifications.success('Gallery-dl configuration saved')
|
notifications.success('Gallery-dl configuration saved')
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
notifications.error(`Failed to save: ${error.message}`)
|
notifications.error(`Failed to save: ${error.message}`)
|
||||||
@@ -524,6 +568,22 @@ async function saveGalleryDLConfig() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function resetGalleryDLConfigToDefaults() {
|
||||||
|
resettingConfig.value = true
|
||||||
|
try {
|
||||||
|
const config = await settingsStore.resetGalleryDLConfig()
|
||||||
|
galleryDLConfigText.value = JSON.stringify(config, null, 2)
|
||||||
|
configIsDefault.value = false
|
||||||
|
configError.value = ''
|
||||||
|
resetConfigDialog.value = false
|
||||||
|
notifications.success('Gallery-dl configuration reset to defaults')
|
||||||
|
} catch (error) {
|
||||||
|
notifications.error(`Failed to reset: ${error.message}`)
|
||||||
|
} finally {
|
||||||
|
resettingConfig.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadPlatforms() {
|
async function loadPlatforms() {
|
||||||
try {
|
try {
|
||||||
const response = await platformsApi.list()
|
const response = await platformsApi.list()
|
||||||
|
|||||||
Reference in New Issue
Block a user