From c2a2162c8687844132b08357b1aadbd18a2056b0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 21 Apr 2026 07:34:33 -0400 Subject: [PATCH] fix(downloader): don't misclassify tier-gated runs with per-item video 403s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a Patreon source loses tier access, gallery-dl emits "Not allowed to view post N" warnings for every post AND yt-dlp tries to fetch HLS manifests which also 403. The ytdl error text contained "HTTP Error 403: Forbidden", which the classifier's ACCESS_DENIED_PATTERNS matched against the full combined stdout+stderr — so the run was labeled ACCESS_DENIED instead of TIER_LIMITED. Two fixes in one edit, because they're interlocked: - Compute per-item vs source-level error lines upfront. has_actual_error now reflects only source-level errors (previously the per-item exclusion was gated on skip evidence, which tier-limited runs don't produce since no content was accessed). - Strip per-item error lines from `combined` before downstream pattern matching so noise from recovered per-item failures doesn't latch onto ACCESS_DENIED / HTTP_ERROR / NOT_FOUND classifiers. Regression test: tier-gated Patreon run with yt-dlp HLS 403s → TIER_LIMITED, not ACCESS_DENIED. Co-Authored-By: Claude Opus 4.7 --- backend/app/services/gallery_dl.py | 78 +++++++++++-------- .../services/test_gallery_dl_categorize.py | 30 +++++++ 2 files changed, 75 insertions(+), 33 deletions(-) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 833da2d..b79d416 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -437,40 +437,52 @@ class GalleryDLService: "download failed", "extraction failed", ] - has_actual_error = any(err in combined for err in actual_error_indicators) - # Non-fatal error lines gallery-dl logs but recovers from. These shouldn't - # block skip detection when the overall download proceeded successfully: - # - "not allowed to view" / "unable to get post": per-item paywall warnings - # on Patreon /c/user/posts URLs where some posts are paywalled. - # - "failed to extract campaign id": logged when the vanity-URL HTML path - # fails, but gallery-dl falls back to the /cw/ redirect and - # recovers the campaign_id on its own. - # - "failed to download": per-item download failure (e.g., a single - # Patreon attachment whose media URL expired / was deleted). The - # downloader logs [download][error] and moves to the next item. - if has_actual_error and (skip_line_count > 0 or has_skip_text): - per_item_patterns = [ - "not allowed to view", - "unable to get post", - "failed to extract campaign id", - "failed to download", - # yt-dlp/youtube-dl missing: each video that gallery-dl tries to - # download fails, but image/skip activity continues. Treat as - # per-item so it doesn't mask TIER_LIMITED/NO_NEW_CONTENT. - # Surface separately as a setup gap (see Dockerfile yt-dlp install). - "cannot import yt-dlp", - "cannot import youtube-dl", - ] - error_lines = [ - line for line in combined.split('\n') - if any(ind in line for ind in actual_error_indicators) - ] - if error_lines and all( - any(p in line for p in per_item_patterns) - for line in error_lines - ): - has_actual_error = False + # Non-fatal error lines gallery-dl logs but recovers from. These are + # per-item failures the run already moved past — they shouldn't cause + # source-level misclassification: + # - "not allowed to view" / "unable to get post": per-item paywall + # warnings on Patreon URLs with mixed tier access. + # - "failed to extract campaign id": vanity-URL HTML path fails, + # gallery-dl falls back to /cw/ and recovers. + # - "failed to download": per-item media failure (expired URL, + # yt-dlp HLS 403 on tier-gated video, deleted attachment). + # Downloader logs [download][error] and continues. + # - "cannot import yt-dlp"/"youtube-dl": video backend missing; + # surfaces via run_stats as a setup gap, not a source failure. + per_item_patterns = [ + "not allowed to view", + "unable to get post", + "failed to extract campaign id", + "failed to download", + "cannot import yt-dlp", + "cannot import youtube-dl", + ] + all_lines = combined.split('\n') + error_lines = [ + line for line in all_lines + if any(ind in line for ind in actual_error_indicators) + ] + per_item_error_lines = [ + line for line in error_lines + if any(p in line for p in per_item_patterns) + ] + source_level_error_lines = [ + line for line in error_lines + if line not in set(per_item_error_lines) + ] + + # Only source-level errors block skip/tier-limited classification. + # Per-item failures are tracked separately in run_stats. + has_actual_error = bool(source_level_error_lines) + + # Strip per-item error lines from the text used for downstream pattern + # matching. Otherwise noise like "HTTP Error 403: Forbidden" from a + # yt-dlp HLS failure latches onto ACCESS_DENIED and masks the real + # signal (tier gating, no new content, etc.). + if per_item_error_lines: + per_item_set = set(per_item_error_lines) + combined = '\n'.join(line for line in all_lines if line not in per_item_set) # If we have skip evidence and no real errors, it's no_new_content if (skip_line_count > 0 or has_skip_text) and not has_actual_error: diff --git a/backend/tests/services/test_gallery_dl_categorize.py b/backend/tests/services/test_gallery_dl_categorize.py index e77e24f..35d79c4 100644 --- a/backend/tests/services/test_gallery_dl_categorize.py +++ b/backend/tests/services/test_gallery_dl_categorize.py @@ -163,6 +163,36 @@ def test_skip_activity_takes_precedence_over_tier_limited(): assert error_type == ErrorType.NO_NEW_CONTENT +def test_tier_limited_with_ytdl_403s_not_access_denied(): + """Real-world Patreon scenario: account lost tier access. Posts show up as + '[patreon][warning] Not allowed to view post N' (no downloads attempted) + AND yt-dlp tries to fetch video HLS manifests that are also tier-gated, + producing '[downloader.ytdl][error] HTTP Error 403: Forbidden' per item. + + Before the fix, the 'Forbidden' text in per-item ytdl errors latched onto + ACCESS_DENIED_PATTERNS and misclassified the whole source as ACCESS_DENIED. + Correct behavior: TIER_LIMITED (the warnings are the dominant signal; the + per-item 403s are just tier gating manifesting at the HLS layer).""" + service = _make_service() + tier_warnings = "\n".join( + f"[patreon][warning] Not allowed to view post {100000 + i}" + for i in range(20) + ) + ytdl_errors = "\n".join( + "[downloader.ytdl][error] ExtractorError: Failed to download m3u8 " + "information: HTTP Error 403: Forbidden " + f"(caused by ) ({i}/4)" + for i in range(1, 5) + ) + "\n[download][error] Failed to download 02_video.mp4\n" + stderr = tier_warnings + "\n" + ytdl_errors + stdout = "" + + error_type, message = service._categorize_error(1, stdout, stderr) + + assert error_type == ErrorType.TIER_LIMITED, f"got {error_type}: {message}" + assert "20" in message + + def test_ytdl_missing_with_skips_classifies_as_no_new_content(): """Real-world scenario from a Patreon source: container is missing yt-dlp so every video logs '[downloader.ytdl][error] Cannot import yt-dlp' plus