feat(downloader): tier-limited classification, config defaults, yt-dlp support

- Classify Patreon "Not allowed to view post" warnings as TIER_LIMITED so
  subscription-gated runs are distinguished from genuine failures, and persist
  error_type/message on completed runs so the distinction survives to the UI.
- Fold PLATFORM_DEFAULTS into _get_default_config so gallery-dl.conf is a
  complete, editable document; _build_config_for_source now preserves the
  user's conf and only re-seeds missing platform sections.
- Add Reset-to-Defaults action in Settings (vs. Revert Changes which just
  reloads disk); show info banner when no gallery-dl.conf exists yet.
- Fix Discord embeds option: must be "all" string, not bool (gallery-dl
  iterates the value).
- Install yt-dlp + ffmpeg in Docker images so Patreon/Mux HLS video posts
  download instead of logging "Cannot import yt-dlp" and skipping.
- Recognize yt-dlp import failures as per-item errors so they don't mask
  TIER_LIMITED/NO_NEW_CONTENT classification.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

backend/Dockerfile.dev

@@ -6,6 +6,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc \
     libpq-dev \
+    ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Python dependencies

backend/app/services/gallery_dl.py

@@ -29,6 +29,7 @@ class ErrorType(str, Enum):
     ACCESS_DENIED = "access_denied"
     NETWORK_ERROR = "network_error"
     NO_NEW_CONTENT = "no_new_content"
+    TIER_LIMITED = "tier_limited"
     TIMEOUT = "timeout"
     HTTP_ERROR = "http_error"
     UNSUPPORTED_URL = "unsupported_url"
@@ -229,8 +230,10 @@ class GalleryDLService:
             "content_types": ["all"],
             "directory": ["{channel[name]}"],
             "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
-            # Additional options
-            "embeds": True,  # Download embedded URLs
+            # Additional options. gallery-dl's discord extractor treats `embeds`
+            # as an iterable of embed types (strings) or the string "all" — a
+            # bool causes "argument of type 'bool' is not iterable" at runtime.
+            "embeds": "all",
             "stickers": True,  # Download stickers
             "reactions": False,  # Skip reaction images
             # Deep crawling options
@@ -281,8 +284,16 @@ class GalleryDLService:
         return self._get_default_config()
 
     def _get_default_config(self) -> dict:
-        """Get default gallery-dl configuration."""
-        return {
+        """Get default gallery-dl configuration.
+
+        Platform-specific defaults from PLATFORM_DEFAULTS are folded into
+        extractor.<platform> so the returned dict is a complete, self-contained
+        gallery-dl config — editable via the Settings view and writable to
+        gallery-dl.conf for full user control. The internal `content_types`
+        meta-key is stripped (it's our abstraction, not a gallery-dl option —
+        it gets translated per-platform to `files`/`include` at run time).
+        """
+        config = {
             "extractor": {
                 "base-directory": self.settings.download_path,
                 "archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
@@ -312,6 +323,11 @@ class GalleryDLService:
                 "progress": True,
             },
         }
+        for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
+            config["extractor"][platform_name] = {
+                key: value for key, value in defaults.items() if key != "content_types"
+            }
+        return config
 
     def _build_config_for_source(
         self,
@@ -321,30 +337,33 @@ class GalleryDLService:
     ) -> dict:
         """Build a merged configuration for a specific source.
 
-        Merges: base config → platform defaults → source overrides
+        Precedence: _get_default_config() (built-in) → gallery-dl.conf on disk
+        → per-source overrides. The platform's extractor section is preserved
+        from the base config (deep copy) and source overrides are applied on
+        top — so user tweaks in gallery-dl.conf survive. If the user's conf
+        dropped the platform section entirely, re-seed from defaults so
+        platform behavior stays sticky.
         """
         config = json.loads(json.dumps(self._base_config))  # Deep copy
 
-        # Get platform defaults
-        platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {})
-
-        # Ensure extractor section exists
         if "extractor" not in config:
             config["extractor"] = {}
 
-        # Apply base directory
-        config["extractor"]["base-directory"] = destination
+        # Re-seed platform defaults if the user's conf is missing the section.
+        # Prevents "I deleted extractor.discord from my conf and silently lost
+        # embeds/stickers/threads." from becoming a footgun.
+        if platform not in config["extractor"]:
+            seed = self._get_default_config()
+            config["extractor"][platform] = seed.get("extractor", {}).get(platform, {})
 
-        # Apply rate limiting overrides
+        # extractor.* global overrides from source config
+        config["extractor"]["base-directory"] = destination
         if source_config.sleep is not None:
             config["extractor"]["sleep"] = source_config.sleep
         if source_config.sleep_request is not None:
             config["extractor"]["sleep-request"] = source_config.sleep_request
-
-        # Apply skip setting
         config["extractor"]["skip"] = source_config.skip_existing
 
-        # Configure metadata postprocessor
         if source_config.save_metadata:
             config["extractor"]["postprocessors"] = [
                 {
@@ -357,43 +376,29 @@ class GalleryDLService:
         else:
             config["extractor"].pop("postprocessors", None)
 
-        # Build platform-specific section
-        platform_config = {}
+        # Source overrides on the platform-specific section. We preserve
+        # whatever is in the base config (platform defaults or user's conf)
+        # and only set keys that the source explicitly controls.
+        platform_section = config["extractor"][platform]
 
-        # Apply all platform defaults first (videos, embeds, stickers, etc.)
-        for key, value in platform_defaults.items():
-            if key not in ["content_types", "directory", "filename"]:
-                platform_config[key] = value
-
-        # Content types (platform-specific handling)
+        # Content types translate to platform-specific gallery-dl keys.
         if platform == "patreon":
-            # Patreon uses "files" array for content types
             if "all" in source_config.content_types:
-                platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"]
+                platform_section["files"] = ["images", "image_large", "attachments", "postfile", "content"]
             else:
-                platform_config["files"] = source_config.content_types
+                platform_section["files"] = source_config.content_types
         elif platform == "hentaifoundry":
-            # HentaiFoundry uses "include" for content filtering
             if "pictures" in source_config.content_types or "all" in source_config.content_types:
-                platform_config["include"] = "all"
+                platform_section["include"] = "all"
 
-        # Directory pattern
+        # Directory/filename: source override wins; otherwise inherit whatever
+        # the base config already has (platform default or user-set value).
         if source_config.directory_pattern:
-            platform_config["directory"] = source_config.directory_pattern
-        elif "directory" in platform_defaults:
-            platform_config["directory"] = platform_defaults["directory"]
-
-        # Filename pattern
+            platform_section["directory"] = source_config.directory_pattern
         if source_config.filename_pattern:
-            platform_config["filename"] = source_config.filename_pattern
-        elif "filename" in platform_defaults:
-            platform_config["filename"] = platform_defaults["filename"]
+            platform_section["filename"] = source_config.filename_pattern
 
-        # Always save metadata at platform level too
-        platform_config["metadata"] = source_config.save_metadata
-
-        # Add platform config to extractor
-        config["extractor"][platform] = platform_config
+        platform_section["metadata"] = source_config.save_metadata
 
         return config
 
@@ -450,6 +455,12 @@ class GalleryDLService:
                 "unable to get post",
                 "failed to extract campaign id",
                 "failed to download",
+                # yt-dlp/youtube-dl missing: each video that gallery-dl tries to
+                # download fails, but image/skip activity continues. Treat as
+                # per-item so it doesn't mask TIER_LIMITED/NO_NEW_CONTENT.
+                # Surface separately as a setup gap (see Dockerfile yt-dlp install).
+                "cannot import yt-dlp",
+                "cannot import youtube-dl",
             ]
             error_lines = [
                 line for line in combined.split('\n')
@@ -547,6 +558,23 @@ class GalleryDLService:
             logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
             return ErrorType.NO_NEW_CONTENT, "No new content to download"
 
+        # Subscription-tier gating: gallery-dl emits [patreon][warning] "Not allowed to
+        # view post N" for posts the current auth token can't access. A run that exits
+        # 1/4 with only these warnings and no downloads means the account's tier doesn't
+        # cover the requested posts — it's a legitimate state, not a failure.
+        if return_code in (1, 4) and not has_actual_error:
+            tier_gated_lines = [
+                line for line in combined.split('\n')
+                if "][warning]" in line and "not allowed to view post" in line
+            ]
+            if tier_gated_lines:
+                count = len(tier_gated_lines)
+                logger.debug(f"Error classified as tier_limited via: rc{return_code} + {count} tier-gated warnings")
+                return (
+                    ErrorType.TIER_LIMITED,
+                    f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
+                )
+
         # Final fallback - genuine unknown error
         return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
 
@@ -573,6 +601,8 @@ class GalleryDLService:
           - per_item_failures: [download][error] lines (single-item failures the
             downloader logs but recovers from)
           - warning_count: [*][warning] lines
+          - tier_gated_count: 'not allowed to view post' warnings (Patreon
+            subscription-tier gating — informational, not a failure)
         """
         stdout = stdout or ""
         stderr = stderr or ""
@@ -593,12 +623,18 @@ class GalleryDLService:
             1 for line in stderr.splitlines() if "][warning]" in line.lower()
         )
 
+        tier_gated_count = sum(
+            1 for line in stderr.splitlines()
+            if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
+        )
+
         return {
             "exit_code": return_code,
             "downloaded_count": self._count_downloaded_files(stdout),
             "skipped_count": skipped_stdout + skipped_stderr,
             "per_item_failures": per_item_failures,
             "warning_count": warning_count,
+            "tier_gated_count": tier_gated_count,
         }
 
     @staticmethod
@@ -769,8 +805,10 @@ class GalleryDLService:
                 proc.returncode, proc.stdout, proc.stderr
             )
 
-            # NO_NEW_CONTENT is considered successful
-            success = error_type == ErrorType.NO_NEW_CONTENT
+            # NO_NEW_CONTENT and TIER_LIMITED are non-failure states — the run completed
+            # as well as it could given external constraints (archive up-to-date, or
+            # current subscription tier can't see the requested posts).
+            success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
 
             return DownloadResult(
                 success=success,

backend/app/tasks/downloads.py

@@ -420,6 +420,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
 
             if dl_result.success:
                 download.status = DownloadStatus.COMPLETED
+                # Some success states still carry an informational error_type
+                # (e.g., TIER_LIMITED — run completed, but the subscription tier
+                # blocked posts we'd have otherwise downloaded). Persist it so
+                # the UI can distinguish "fully succeeded" from "succeeded with
+                # external constraints".
+                download.error_type = dl_result.error_type.value if dl_result.error_type else None
+                download.error_message = dl_result.error_message
                 source.last_success = utcnow()
                 source.error_count = 0
                 logger.info(f"Download completed for {subscription_name}/{source_platform}: {dl_result.files_downloaded} files")

backend/requirements.txt

@@ -27,6 +27,12 @@ pyyaml
 # gallery-dl (>=1.31.10 fixes Patreon HTTP 426 / OAuth flow)
 gallery-dl>=1.31.10
 
+# yt-dlp: video backend gallery-dl shells out to for Patreon/Mux HLS streams,
+# embedded YouTube, etc. Without it, video posts log "[downloader.ytdl][error]
+# Cannot import yt-dlp" and skip. Requires ffmpeg in the system image to mux
+# HLS segments into mp4.
+yt-dlp
+
 # Testing
 pytest
 pytest-asyncio

backend/tests/services/test_gallery_dl_categorize.py

@@ -93,3 +93,94 @@ def test_patreon_per_item_download_failure_without_skips_still_fails():
     error_type, _message = service._categorize_error(1, stdout, stderr)
 
     assert error_type != ErrorType.NO_NEW_CONTENT
+
+
+def test_patreon_tier_gated_warnings_classify_as_tier_limited():
+    """Many 'Not allowed to view post N' warnings + exit 4 + no downloads →
+    TIER_LIMITED. Reflects a Patreon tier downgrade where older posts are now
+    paywalled but we don't want to call this a failure."""
+    service = _make_service()
+    stderr = "\n".join(
+        f"[patreon][warning] Not allowed to view post {100000 + i}"
+        for i in range(15)
+    )
+    stdout = ""
+
+    error_type, message = service._categorize_error(4, stdout, stderr)
+
+    assert error_type == ErrorType.TIER_LIMITED
+    assert "15" in message
+
+
+def test_tier_limited_with_exit_code_1_also_classifies():
+    """Mixed run: exit 1 with tier-gated warnings and no real errors."""
+    service = _make_service()
+    stderr = "[patreon][warning] Not allowed to view post 999\n"
+    stdout = ""
+
+    error_type, _message = service._categorize_error(1, stdout, stderr)
+
+    assert error_type == ErrorType.TIER_LIMITED
+
+
+def test_tier_limited_not_applied_when_real_error_present():
+    """Tier warnings + a real auth error → auth error wins (TIER_LIMITED is
+    only for 'everything else was fine' runs)."""
+    service = _make_service()
+    stderr = (
+        "[patreon][warning] Not allowed to view post 999\n"
+        '[patreon][error] "GET /api/posts HTTP/1.1" 401 None\n'
+    )
+    stdout = ""
+
+    error_type, _message = service._categorize_error(1, stdout, stderr)
+
+    assert error_type == ErrorType.AUTH_ERROR
+
+
+def test_tier_limited_does_not_fire_on_exit_0():
+    """Successful run with some paywalled posts as warnings isn't an error at
+    all — don't reach the TIER_LIMITED branch (exit 0 → success path earlier)."""
+    service = _make_service()
+    stderr = "[patreon][warning] Not allowed to view post 999\n"
+    stdout = "/data/downloads/Artist/patreon/post1/img.jpg\n"
+
+    error_type, _message = service._categorize_error(0, stdout, stderr)
+
+    # Exit 0 + stdout activity should not hit the TIER_LIMITED branch
+    assert error_type != ErrorType.TIER_LIMITED
+
+
+def test_skip_activity_takes_precedence_over_tier_limited():
+    """If there's also skip activity, NO_NEW_CONTENT wins — precedence matters
+    because the tier_limited branch is the last informational fallback."""
+    service = _make_service()
+    stderr = "[patreon][warning] Not allowed to view post 999\n"
+    stdout = "# /data/downloads/Artist/patreon/post1/img.jpg\n"
+
+    error_type, _message = service._categorize_error(1, stdout, stderr)
+
+    assert error_type == ErrorType.NO_NEW_CONTENT
+
+
+def test_ytdl_missing_with_skips_classifies_as_no_new_content():
+    """Real-world scenario from a Patreon source: container is missing yt-dlp
+    so every video logs '[downloader.ytdl][error] Cannot import yt-dlp' plus
+    one [download][error] per video. With overwhelming skip activity, this
+    should classify as NO_NEW_CONTENT (the setup gap is per-item — it
+    shouldn't mask the dominant signal that the archive is up-to-date).
+
+    Regression: previously the yt-dlp import error didn't match any per-item
+    pattern, so has_actual_error stayed True and the run was misclassified as
+    UNKNOWN_ERROR even though 1300+ files were skipped."""
+    service = _make_service()
+    stderr = (
+        "[downloader.ytdl][error] Cannot import yt-dlp or youtube-dl\n"
+        "[download][error] Failed to download 02_video.mp4\n"
+        "[download][error] Failed to download 02_video.mp4\n"
+    )
+    stdout = "# /data/downloads/Artist/patreon/post/01_img.jpg\n" * 100
+
+    error_type, _message = service._categorize_error(4, stdout, stderr)
+
+    assert error_type == ErrorType.NO_NEW_CONTENT

backend/tests/services/test_gallery_dl_config_merge.py

@@ -0,0 +1,160 @@
+"""Unit tests for the unified gallery-dl config pipeline.
+
+Covers two things:
+  1. `_get_default_config()` now includes per-platform defaults under
+     `extractor.<platform>` so the generated config is self-contained and
+     editable via the Settings view / gallery-dl.conf.
+  2. `_build_config_for_source` deep-merges user overrides: the platform
+     section from `self._base_config` survives, source-specific keys
+     (content_types → files/include, directory_pattern, filename_pattern,
+     sleep, skip, metadata) are applied on top, and missing platform
+     sections are re-seeded from defaults so behavior stays sticky.
+"""
+from unittest.mock import patch
+
+from app.services.gallery_dl import GalleryDLService, SourceConfig
+
+
+def _make_service(base_config: dict = None):
+    """Build a service with an optional base config (bypasses disk IO)."""
+    with patch.object(GalleryDLService, "_load_base_config", return_value=base_config or {}):
+        return GalleryDLService(rate_limit=1.0)
+
+
+def test_default_config_includes_platform_sections():
+    """_get_default_config() bakes PLATFORM_DEFAULTS into extractor.<platform>."""
+    service = _make_service()
+    cfg = service._get_default_config()
+
+    assert "extractor" in cfg
+    for platform in ("patreon", "discord", "subscribestar", "hentaifoundry", "pixiv", "deviantart"):
+        assert platform in cfg["extractor"], f"{platform} missing from default config"
+
+
+def test_default_config_strips_content_types_meta_key():
+    """`content_types` is an internal abstraction — must not leak into the
+    gallery-dl config (it gets translated to `files`/`include` per platform)."""
+    service = _make_service()
+    cfg = service._get_default_config()
+
+    for platform_section in cfg["extractor"].values():
+        if isinstance(platform_section, dict):
+            assert "content_types" not in platform_section
+
+
+def test_discord_embeds_is_not_bool_regression():
+    """Regression: `embeds: True` caused `argument of type 'bool' is not
+    iterable` at runtime. Must be a string or list (the gallery-dl discord
+    extractor iterates it to filter embed types)."""
+    service = _make_service()
+    cfg = service._get_default_config()
+
+    embeds = cfg["extractor"]["discord"]["embeds"]
+    assert not isinstance(embeds, bool), f"discord.embeds must not be bool (got {embeds!r})"
+    assert isinstance(embeds, (str, list))
+
+
+def test_build_config_preserves_user_overrides_on_platform_section():
+    """If the user set extractor.discord.embeds to a custom value, it must
+    survive _build_config_for_source (that's the whole point of unification)."""
+    user_config = {
+        "extractor": {
+            "discord": {"embeds": ["image"], "stickers": False},
+        },
+    }
+    service = _make_service(user_config)
+    result = service._build_config_for_source(
+        platform="discord",
+        source_config=SourceConfig(content_types=["all"]),
+        destination="/data/downloads/Me/discord",
+    )
+    assert result["extractor"]["discord"]["embeds"] == ["image"]
+    assert result["extractor"]["discord"]["stickers"] is False
+
+
+def test_build_config_reseeds_platform_when_missing_from_user_conf():
+    """If user's conf has no discord section, re-inject defaults so
+    behavior stays sticky (deleting extractor.discord shouldn't silently
+    break downloads)."""
+    user_config = {"extractor": {"patreon": {"files": []}}}  # discord missing
+    service = _make_service(user_config)
+    result = service._build_config_for_source(
+        platform="discord",
+        source_config=SourceConfig(content_types=["all"]),
+        destination="/data/downloads/Me/discord",
+    )
+
+    discord_section = result["extractor"]["discord"]
+    # Re-seeded defaults: embeds, stickers, threads, directory, filename
+    assert "embeds" in discord_section
+    assert "stickers" in discord_section
+    assert "threads" in discord_section
+
+
+def test_source_config_overrides_win_over_user_conf():
+    """Per-source settings (directory_pattern, filename_pattern, sleep) must
+    win over user conf — source-specific is more specific than global."""
+    user_config = {
+        "extractor": {
+            "discord": {"directory": ["global_pattern"], "filename": "{id}.{extension}"},
+        },
+    }
+    service = _make_service(user_config)
+    result = service._build_config_for_source(
+        platform="discord",
+        source_config=SourceConfig(
+            content_types=["all"],
+            directory_pattern=["source_pattern"],
+            filename_pattern="{date}_{id}.{extension}",
+            sleep=5.0,
+        ),
+        destination="/data/downloads/Me/discord",
+    )
+
+    assert result["extractor"]["discord"]["directory"] == ["source_pattern"]
+    assert result["extractor"]["discord"]["filename"] == "{date}_{id}.{extension}"
+    assert result["extractor"]["sleep"] == 5.0
+
+
+def test_build_config_preserves_user_extractor_globals():
+    """User-set extractor-level keys (e.g., retries, timeout, custom archive
+    path) must survive unless overridden by source config."""
+    user_config = {
+        "extractor": {
+            "retries": 10,
+            "timeout": 60.0,
+            "archive": "/custom/archive.db",
+        },
+    }
+    service = _make_service(user_config)
+    result = service._build_config_for_source(
+        platform="patreon",
+        source_config=SourceConfig(content_types=["all"]),
+        destination="/data/downloads/Me/patreon",
+    )
+
+    assert result["extractor"]["retries"] == 10
+    assert result["extractor"]["timeout"] == 60.0
+    assert result["extractor"]["archive"] == "/custom/archive.db"
+
+
+def test_build_config_patreon_content_types_translated_to_files():
+    """Legacy behavior preserved: patreon content_types → files list."""
+    service = _make_service()
+    result = service._build_config_for_source(
+        platform="patreon",
+        source_config=SourceConfig(content_types=["images", "attachments"]),
+        destination="/data/downloads/Me/patreon",
+    )
+    assert result["extractor"]["patreon"]["files"] == ["images", "attachments"]
+
+
+def test_build_config_hentaifoundry_content_types_translated_to_include():
+    """Legacy behavior preserved: hentaifoundry 'all' content_type → include."""
+    service = _make_service()
+    result = service._build_config_for_source(
+        platform="hentaifoundry",
+        source_config=SourceConfig(content_types=["all"]),
+        destination="/data/downloads/Me/hentaifoundry",
+    )
+    assert result["extractor"]["hentaifoundry"]["include"] == "all"

backend/tests/services/test_gallery_dl_run_stats.py

@@ -22,6 +22,7 @@ def test_run_stats_all_zeros_on_empty_output():
         "skipped_count": 0,
         "per_item_failures": 0,
         "warning_count": 0,
+        "tier_gated_count": 0,
     }
 
 
@@ -79,3 +80,18 @@ def test_run_stats_exit_code_passed_through():
     assert service._compute_run_stats(0, "", "")["exit_code"] == 0
     assert service._compute_run_stats(4, "", "")["exit_code"] == 4
     assert service._compute_run_stats(-9, "", "")["exit_code"] == -9
+
+
+def test_run_stats_tier_gated_count_from_patreon_warnings():
+    """'Not allowed to view post N' warnings tallied separately from generic warnings."""
+    service = _make_service()
+    stderr = (
+        "[patreon][warning] Not allowed to view post 100\n"
+        "[patreon][warning] Not allowed to view post 101\n"
+        "[patreon][warning] Not allowed to view post 102\n"
+        "[downloader.http][warning] '404 OK' for 'https://example.com/x'\n"
+    )
+    stats = service._compute_run_stats(4, "", stderr)
+    assert stats["tier_gated_count"] == 3
+    # Still tallied in warning_count — tier_gated is a subset, not separate
+    assert stats["warning_count"] == 4