From 66ff671f091b95fc0d3deb904ce432fe4e3b03c8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 19:00:50 -0400 Subject: [PATCH] fix(downloads): forward Patreon Referer/Origin to yt-dlp for Mux videos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged 2026-06-01 (DaferQ patreon, event #38919). Video posts hosted on Mux carry a JWT that encodes a playback restriction policy (`playback_restriction_id` in the token). The token signature alone is not sufficient — Mux's CDN ALSO checks Referer/Origin on every fetch. gallery-dl's HEAD probe to `stream.mux.com/.m3u8?token=...` returns 200 (token is valid, HEAD sends no Referer that Mux's policy rejects), but when yt-dlp follows up with the actual manifest GET it sends its own default Referer and Mux 403s. yt-dlp retries 4 times, gives up, the single video fails — every other image in the same post still downloads. Static `downloader.ytdl.raw-options.http_headers` block now pins Referer and Origin to `https://www.patreon.com` so yt-dlp's manifest fetch clears the policy check. Headers-only restrictions are now handled. Mux IP-range restrictions (if a creator opts into them) remain unfixable from our worker — those would need a Patreon-region IP. Per-video PARTIAL classifier (shipped in plan #544 last commit) already handles the residual case: a run that grabs all images and fails 1 video classifies as status=ok rather than flipping the whole event red. --- backend/app/services/gallery_dl.py | 19 +++++++++++++++++++ tests/test_gallery_dl_service.py | 11 +++++++++++ 2 files changed, 30 insertions(+) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 67fc59b..0f19793 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -257,6 +257,25 @@ class GalleryDLService: "part-directory": str(self._config_dir / "temp"), "retries": 3, "timeout": 120.0, + # Forward Patreon as Referer/Origin to yt-dlp when it + # fetches video manifests. Operator-flagged 2026-06-01 + # (DaferQ patreon): video posts hosted on Mux carry a JWT + # playback restriction that checks Referer/Origin on every + # request — not just the token signature. gallery-dl's + # HEAD probe to stream.mux.com returns 200 (the token is + # valid), but yt-dlp's actual GET-with-Range to fetch the + # m3u8 manifest 403s because yt-dlp sends its own default + # Referer, which Mux's policy rejects. Forcing the right + # headers fixes the headers-only case; Mux IP-range + # restrictions are unfixable from here. + "ytdl": { + "raw-options": { + "http_headers": { + "Referer": "https://www.patreon.com/", + "Origin": "https://www.patreon.com", + }, + }, + }, }, "output": {"progress": True}, } diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index ef196b5..d3607a5 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -315,3 +315,14 @@ def test_categorize_tier_limited_wins_over_partial(gdl): return_code=1, stdout=stdout, stderr=stderr, ) assert etype == ErrorType.TIER_LIMITED + + +def test_default_config_forwards_patreon_referer_to_ytdl(gdl): + """Operator-flagged 2026-06-01: Mux video playback restrictions reject + yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The + fix is a static `downloader.ytdl.raw-options.http_headers` block, so + it's enough to assert the keys land in the default config.""" + cfg = gdl._get_default_config() + headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"] + assert headers["Referer"] == "https://www.patreon.com/" + assert headers["Origin"] == "https://www.patreon.com"