Files
FabledCurator/tests/test_download_backends.py
T
bvandeusenandClaude Opus 5.5 84e5448941
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
feat: Discord on the native core ingester — client, downloader, ledgers, wiring (milestone 428)
Discord was the last focus platform still on gallery-dl. This adds the native
path, mirrored from gallery-dl 1.32.13's discord extractor:

- discord_client: API v10 with the user token and gallery-dl's request
  profile (dated Firefox UA, Referer). Walks a server, category, forum,
  channel or thread in gallery-dl's order and pages each channel newest-first.
  Files are attachments, then embeds, then forwards, numbered across the
  message. The resume cursor is <channel>:<before>. Text-only messages are
  not posts, since gallery-dl never made them.
- discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans
  names on Linux (only `/` and control characters change), so existing files
  are skipped_disk rather than fetched again. Sidecars carry identity only.
  The message record keeps gallery-dl's keys, so parse_sidecar,
  derive_post_url and the drop grouping read it unchanged.
- The ledger keys on the attachment id (or a hash of an embed's URL path),
  not the file's position, which an edit can renumber. Migration 0111.
- DiscordIngester: token auth, body canary off (files-only drops are
  normal). Registered as native, verified by token, and serialised
  per-platform, since every source shares one user token.
- ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out
  on a multi-channel source now ends the quiet channel, not the whole walk.
  Clients without the seam behave as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-24 19:42:00 -04:00

138 lines
5.0 KiB
Python

"""download_backends — the single predicate that routes a platform to the
native ingester vs. gallery-dl. Pure, no DB."""
from pathlib import Path
import pytest
from backend.app.services.download_backends import (
NATIVE_INGESTER_PLATFORMS,
_native_ingester_cls,
_unsupported_platform_message,
run_download,
uses_native_ingester,
verify_source_credential,
)
from backend.app.services.gallery_dl import ErrorType
def test_native_platforms():
for platform in ("patreon", "subscribestar", "discord"):
assert uses_native_ingester(platform) is True
assert platform in NATIVE_INGESTER_PLATFORMS
def test_pixiv_is_no_longer_native():
"""Retired at milestone #406 — unregistered in phase 1, deleted in phase 2.
The refusal below is what stops it falling through to gallery-dl now that
it is neither native nor registered."""
assert uses_native_ingester("pixiv") is False
assert "pixiv" not in NATIVE_INGESTER_PLATFORMS
# --- the retired-platform guard (#406 phase 1) -----------------------------
class _RecordingGalleryDL:
"""Stands in for GalleryDLService: records whether a download was attempted."""
def __init__(self):
self.calls = []
async def download(self, **kwargs):
self.calls.append(kwargs["platform"])
return "reached gallery-dl"
def _ctx(platform):
return {
"platform": platform, "url": f"https://example.invalid/{platform}",
"artist_slug": "someone", "cookies_path": None, "auth_token": None,
}
@pytest.mark.asyncio
async def test_a_retired_platform_never_reaches_a_downloader():
"""An enabled source on a retired platform is data that survives a deploy.
Unguarded, pixiv — no longer native — would fall straight through to the
gallery-dl branch, which still has a pixiv extractor."""
gdl = _RecordingGalleryDL()
result, campaign_id = await run_download(
ctx=_ctx("pixiv"), source_config=None, skip_value=False, mode=None,
gdl=gdl, sync_session_factory=None,
)
assert gdl.calls == []
assert result.success is False
assert result.error_type == ErrorType.UNSUPPORTED_URL
assert "pixiv" in result.error_message
assert campaign_id is None
@pytest.mark.asyncio
async def test_a_supported_gallery_dl_platform_still_reaches_gallery_dl():
"""The positive control. Without it, a guard that refused EVERY platform
would pass the test above just as well (rule #167)."""
gdl = _RecordingGalleryDL()
result, _ = await run_download(
ctx=_ctx("hentaifoundry"), source_config=None, skip_value=False, mode=None,
gdl=gdl, sync_session_factory=None,
)
assert gdl.calls == ["hentaifoundry"]
assert result == "reached gallery-dl"
def test_the_guard_discriminates_by_registration():
assert _unsupported_platform_message("hentaifoundry") is None
assert _unsupported_platform_message("patreon") is None
assert _unsupported_platform_message("pixiv") is not None
assert _unsupported_platform_message("deviantart") is not None
@pytest.mark.asyncio
async def test_verifying_a_retired_platform_is_inconclusive_not_rejected():
"""Nothing is probed, so nothing is rejected — returning False would tell the
operator their credential is bad when the platform is simply gone."""
ok, message = await verify_source_credential(
platform="pixiv", url="https://www.pixiv.net/users/1", artist_slug="someone",
config_overrides=None, cookies_path=None, auth_token=None,
images_root=Path("/nonexistent"),
)
assert ok is None
assert "pixiv" in message
def test_gallery_dl_platforms_are_not_native():
# The platforms still served by gallery-dl must NOT route to the native
# ingester — guards an accidental over-broad migration.
assert uses_native_ingester("hentaifoundry") is False
@pytest.mark.asyncio
async def test_discord_verify_without_a_token_is_a_rejection_not_a_request():
"""Discord authenticates by token (milestone 428); with none saved there is
nothing to send, and saying so beats an HTTP 401 from Discord."""
ok, message = await verify_source_credential(
platform="discord", url="https://discord.com/channels/1/2",
artist_slug="someone", config_overrides=None, cookies_path=None,
auth_token=None, images_root=Path("/nonexistent"),
)
assert ok is False
assert "token" in message.lower()
def test_unknown_platform_is_not_native():
assert uses_native_ingester("nonsense") is False
def test_every_native_platform_dispatches_to_an_ingester():
"""The dispatch table and NATIVE_INGESTER_PLATFORMS have to agree, or a
platform that routes native raises KeyError mid-download instead of being
refused up front. Written over the set rather than per-platform so adding
one to NATIVE_INGESTER_PLATFORMS and forgetting the class fails here.
(This replaces the per-platform dispatch tests, one of which was pixiv's;
it was deleted with pixiv at milestone #406 phase 2.)"""
for platform in NATIVE_INGESTER_PLATFORMS:
assert _native_ingester_cls(platform) is not None