feat(downloader): tier-limited classification, config defaults, yt-dlp support
- Classify Patreon "Not allowed to view post" warnings as TIER_LIMITED so subscription-gated runs are distinguished from genuine failures, and persist error_type/message on completed runs so the distinction survives to the UI. - Fold PLATFORM_DEFAULTS into _get_default_config so gallery-dl.conf is a complete, editable document; _build_config_for_source now preserves the user's conf and only re-seeds missing platform sections. - Add Reset-to-Defaults action in Settings (vs. Revert Changes which just reloads disk); show info banner when no gallery-dl.conf exists yet. - Fix Discord embeds option: must be "all" string, not bool (gallery-dl iterates the value). - Install yt-dlp + ffmpeg in Docker images so Patreon/Mux HLS video posts download instead of logging "Cannot import yt-dlp" and skipping. - Recognize yt-dlp import failures as per-item errors so they don't mask TIER_LIMITED/NO_NEW_CONTENT classification. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -29,6 +29,7 @@ class ErrorType(str, Enum):
|
||||
ACCESS_DENIED = "access_denied"
|
||||
NETWORK_ERROR = "network_error"
|
||||
NO_NEW_CONTENT = "no_new_content"
|
||||
TIER_LIMITED = "tier_limited"
|
||||
TIMEOUT = "timeout"
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
@@ -229,8 +230,10 @@ class GalleryDLService:
|
||||
"content_types": ["all"],
|
||||
"directory": ["{channel[name]}"],
|
||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||||
# Additional options
|
||||
"embeds": True, # Download embedded URLs
|
||||
# Additional options. gallery-dl's discord extractor treats `embeds`
|
||||
# as an iterable of embed types (strings) or the string "all" — a
|
||||
# bool causes "argument of type 'bool' is not iterable" at runtime.
|
||||
"embeds": "all",
|
||||
"stickers": True, # Download stickers
|
||||
"reactions": False, # Skip reaction images
|
||||
# Deep crawling options
|
||||
@@ -281,8 +284,16 @@ class GalleryDLService:
|
||||
return self._get_default_config()
|
||||
|
||||
def _get_default_config(self) -> dict:
|
||||
"""Get default gallery-dl configuration."""
|
||||
return {
|
||||
"""Get default gallery-dl configuration.
|
||||
|
||||
Platform-specific defaults from PLATFORM_DEFAULTS are folded into
|
||||
extractor.<platform> so the returned dict is a complete, self-contained
|
||||
gallery-dl config — editable via the Settings view and writable to
|
||||
gallery-dl.conf for full user control. The internal `content_types`
|
||||
meta-key is stripped (it's our abstraction, not a gallery-dl option —
|
||||
it gets translated per-platform to `files`/`include` at run time).
|
||||
"""
|
||||
config = {
|
||||
"extractor": {
|
||||
"base-directory": self.settings.download_path,
|
||||
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
|
||||
@@ -312,6 +323,11 @@ class GalleryDLService:
|
||||
"progress": True,
|
||||
},
|
||||
}
|
||||
for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
|
||||
config["extractor"][platform_name] = {
|
||||
key: value for key, value in defaults.items() if key != "content_types"
|
||||
}
|
||||
return config
|
||||
|
||||
def _build_config_for_source(
|
||||
self,
|
||||
@@ -321,30 +337,33 @@ class GalleryDLService:
|
||||
) -> dict:
|
||||
"""Build a merged configuration for a specific source.
|
||||
|
||||
Merges: base config → platform defaults → source overrides
|
||||
Precedence: _get_default_config() (built-in) → gallery-dl.conf on disk
|
||||
→ per-source overrides. The platform's extractor section is preserved
|
||||
from the base config (deep copy) and source overrides are applied on
|
||||
top — so user tweaks in gallery-dl.conf survive. If the user's conf
|
||||
dropped the platform section entirely, re-seed from defaults so
|
||||
platform behavior stays sticky.
|
||||
"""
|
||||
config = json.loads(json.dumps(self._base_config)) # Deep copy
|
||||
|
||||
# Get platform defaults
|
||||
platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {})
|
||||
|
||||
# Ensure extractor section exists
|
||||
if "extractor" not in config:
|
||||
config["extractor"] = {}
|
||||
|
||||
# Apply base directory
|
||||
config["extractor"]["base-directory"] = destination
|
||||
# Re-seed platform defaults if the user's conf is missing the section.
|
||||
# Prevents "I deleted extractor.discord from my conf and silently lost
|
||||
# embeds/stickers/threads." from becoming a footgun.
|
||||
if platform not in config["extractor"]:
|
||||
seed = self._get_default_config()
|
||||
config["extractor"][platform] = seed.get("extractor", {}).get(platform, {})
|
||||
|
||||
# Apply rate limiting overrides
|
||||
# extractor.* global overrides from source config
|
||||
config["extractor"]["base-directory"] = destination
|
||||
if source_config.sleep is not None:
|
||||
config["extractor"]["sleep"] = source_config.sleep
|
||||
if source_config.sleep_request is not None:
|
||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||
|
||||
# Apply skip setting
|
||||
config["extractor"]["skip"] = source_config.skip_existing
|
||||
|
||||
# Configure metadata postprocessor
|
||||
if source_config.save_metadata:
|
||||
config["extractor"]["postprocessors"] = [
|
||||
{
|
||||
@@ -357,43 +376,29 @@ class GalleryDLService:
|
||||
else:
|
||||
config["extractor"].pop("postprocessors", None)
|
||||
|
||||
# Build platform-specific section
|
||||
platform_config = {}
|
||||
# Source overrides on the platform-specific section. We preserve
|
||||
# whatever is in the base config (platform defaults or user's conf)
|
||||
# and only set keys that the source explicitly controls.
|
||||
platform_section = config["extractor"][platform]
|
||||
|
||||
# Apply all platform defaults first (videos, embeds, stickers, etc.)
|
||||
for key, value in platform_defaults.items():
|
||||
if key not in ["content_types", "directory", "filename"]:
|
||||
platform_config[key] = value
|
||||
|
||||
# Content types (platform-specific handling)
|
||||
# Content types translate to platform-specific gallery-dl keys.
|
||||
if platform == "patreon":
|
||||
# Patreon uses "files" array for content types
|
||||
if "all" in source_config.content_types:
|
||||
platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"]
|
||||
platform_section["files"] = ["images", "image_large", "attachments", "postfile", "content"]
|
||||
else:
|
||||
platform_config["files"] = source_config.content_types
|
||||
platform_section["files"] = source_config.content_types
|
||||
elif platform == "hentaifoundry":
|
||||
# HentaiFoundry uses "include" for content filtering
|
||||
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
||||
platform_config["include"] = "all"
|
||||
platform_section["include"] = "all"
|
||||
|
||||
# Directory pattern
|
||||
# Directory/filename: source override wins; otherwise inherit whatever
|
||||
# the base config already has (platform default or user-set value).
|
||||
if source_config.directory_pattern:
|
||||
platform_config["directory"] = source_config.directory_pattern
|
||||
elif "directory" in platform_defaults:
|
||||
platform_config["directory"] = platform_defaults["directory"]
|
||||
|
||||
# Filename pattern
|
||||
platform_section["directory"] = source_config.directory_pattern
|
||||
if source_config.filename_pattern:
|
||||
platform_config["filename"] = source_config.filename_pattern
|
||||
elif "filename" in platform_defaults:
|
||||
platform_config["filename"] = platform_defaults["filename"]
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
# Always save metadata at platform level too
|
||||
platform_config["metadata"] = source_config.save_metadata
|
||||
|
||||
# Add platform config to extractor
|
||||
config["extractor"][platform] = platform_config
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
|
||||
@@ -450,6 +455,12 @@ class GalleryDLService:
|
||||
"unable to get post",
|
||||
"failed to extract campaign id",
|
||||
"failed to download",
|
||||
# yt-dlp/youtube-dl missing: each video that gallery-dl tries to
|
||||
# download fails, but image/skip activity continues. Treat as
|
||||
# per-item so it doesn't mask TIER_LIMITED/NO_NEW_CONTENT.
|
||||
# Surface separately as a setup gap (see Dockerfile yt-dlp install).
|
||||
"cannot import yt-dlp",
|
||||
"cannot import youtube-dl",
|
||||
]
|
||||
error_lines = [
|
||||
line for line in combined.split('\n')
|
||||
@@ -547,6 +558,23 @@ class GalleryDLService:
|
||||
logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
# Subscription-tier gating: gallery-dl emits [patreon][warning] "Not allowed to
|
||||
# view post N" for posts the current auth token can't access. A run that exits
|
||||
# 1/4 with only these warnings and no downloads means the account's tier doesn't
|
||||
# cover the requested posts — it's a legitimate state, not a failure.
|
||||
if return_code in (1, 4) and not has_actual_error:
|
||||
tier_gated_lines = [
|
||||
line for line in combined.split('\n')
|
||||
if "][warning]" in line and "not allowed to view post" in line
|
||||
]
|
||||
if tier_gated_lines:
|
||||
count = len(tier_gated_lines)
|
||||
logger.debug(f"Error classified as tier_limited via: rc{return_code} + {count} tier-gated warnings")
|
||||
return (
|
||||
ErrorType.TIER_LIMITED,
|
||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||
)
|
||||
|
||||
# Final fallback - genuine unknown error
|
||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||
|
||||
@@ -573,6 +601,8 @@ class GalleryDLService:
|
||||
- per_item_failures: [download][error] lines (single-item failures the
|
||||
downloader logs but recovers from)
|
||||
- warning_count: [*][warning] lines
|
||||
- tier_gated_count: 'not allowed to view post' warnings (Patreon
|
||||
subscription-tier gating — informational, not a failure)
|
||||
"""
|
||||
stdout = stdout or ""
|
||||
stderr = stderr or ""
|
||||
@@ -593,12 +623,18 @@ class GalleryDLService:
|
||||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||||
)
|
||||
|
||||
tier_gated_count = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||||
)
|
||||
|
||||
return {
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": self._count_downloaded_files(stdout),
|
||||
"skipped_count": skipped_stdout + skipped_stderr,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -769,8 +805,10 @@ class GalleryDLService:
|
||||
proc.returncode, proc.stdout, proc.stderr
|
||||
)
|
||||
|
||||
# NO_NEW_CONTENT is considered successful
|
||||
success = error_type == ErrorType.NO_NEW_CONTENT
|
||||
# NO_NEW_CONTENT and TIER_LIMITED are non-failure states — the run completed
|
||||
# as well as it could given external constraints (archive up-to-date, or
|
||||
# current subscription tier can't see the requested posts).
|
||||
success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
|
||||
|
||||
return DownloadResult(
|
||||
success=success,
|
||||
|
||||
@@ -420,6 +420,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
|
||||
if dl_result.success:
|
||||
download.status = DownloadStatus.COMPLETED
|
||||
# Some success states still carry an informational error_type
|
||||
# (e.g., TIER_LIMITED — run completed, but the subscription tier
|
||||
# blocked posts we'd have otherwise downloaded). Persist it so
|
||||
# the UI can distinguish "fully succeeded" from "succeeded with
|
||||
# external constraints".
|
||||
download.error_type = dl_result.error_type.value if dl_result.error_type else None
|
||||
download.error_message = dl_result.error_message
|
||||
source.last_success = utcnow()
|
||||
source.error_count = 0
|
||||
logger.info(f"Download completed for {subscription_name}/{source_platform}: {dl_result.files_downloaded} files")
|
||||
|
||||
Reference in New Issue
Block a user