CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is <channel>:<before>. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
138 lines
5.0 KiB
Python
138 lines
5.0 KiB
Python
"""download_backends — the single predicate that routes a platform to the
|
|
native ingester vs. gallery-dl. Pure, no DB."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from backend.app.services.download_backends import (
|
|
NATIVE_INGESTER_PLATFORMS,
|
|
_native_ingester_cls,
|
|
_unsupported_platform_message,
|
|
run_download,
|
|
uses_native_ingester,
|
|
verify_source_credential,
|
|
)
|
|
from backend.app.services.gallery_dl import ErrorType
|
|
|
|
|
|
def test_native_platforms():
|
|
for platform in ("patreon", "subscribestar", "discord"):
|
|
assert uses_native_ingester(platform) is True
|
|
assert platform in NATIVE_INGESTER_PLATFORMS
|
|
|
|
|
|
def test_pixiv_is_no_longer_native():
|
|
"""Retired at milestone #406 — unregistered in phase 1, deleted in phase 2.
|
|
The refusal below is what stops it falling through to gallery-dl now that
|
|
it is neither native nor registered."""
|
|
assert uses_native_ingester("pixiv") is False
|
|
assert "pixiv" not in NATIVE_INGESTER_PLATFORMS
|
|
|
|
|
|
# --- the retired-platform guard (#406 phase 1) -----------------------------
|
|
|
|
|
|
class _RecordingGalleryDL:
|
|
"""Stands in for GalleryDLService: records whether a download was attempted."""
|
|
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
async def download(self, **kwargs):
|
|
self.calls.append(kwargs["platform"])
|
|
return "reached gallery-dl"
|
|
|
|
|
|
def _ctx(platform):
|
|
return {
|
|
"platform": platform, "url": f"https://example.invalid/{platform}",
|
|
"artist_slug": "someone", "cookies_path": None, "auth_token": None,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_retired_platform_never_reaches_a_downloader():
|
|
"""An enabled source on a retired platform is data that survives a deploy.
|
|
Unguarded, pixiv — no longer native — would fall straight through to the
|
|
gallery-dl branch, which still has a pixiv extractor."""
|
|
gdl = _RecordingGalleryDL()
|
|
result, campaign_id = await run_download(
|
|
ctx=_ctx("pixiv"), source_config=None, skip_value=False, mode=None,
|
|
gdl=gdl, sync_session_factory=None,
|
|
)
|
|
assert gdl.calls == []
|
|
assert result.success is False
|
|
assert result.error_type == ErrorType.UNSUPPORTED_URL
|
|
assert "pixiv" in result.error_message
|
|
assert campaign_id is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_supported_gallery_dl_platform_still_reaches_gallery_dl():
|
|
"""The positive control. Without it, a guard that refused EVERY platform
|
|
would pass the test above just as well (rule #167)."""
|
|
gdl = _RecordingGalleryDL()
|
|
result, _ = await run_download(
|
|
ctx=_ctx("hentaifoundry"), source_config=None, skip_value=False, mode=None,
|
|
gdl=gdl, sync_session_factory=None,
|
|
)
|
|
assert gdl.calls == ["hentaifoundry"]
|
|
assert result == "reached gallery-dl"
|
|
|
|
|
|
def test_the_guard_discriminates_by_registration():
|
|
assert _unsupported_platform_message("hentaifoundry") is None
|
|
assert _unsupported_platform_message("patreon") is None
|
|
assert _unsupported_platform_message("pixiv") is not None
|
|
assert _unsupported_platform_message("deviantart") is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_verifying_a_retired_platform_is_inconclusive_not_rejected():
|
|
"""Nothing is probed, so nothing is rejected — returning False would tell the
|
|
operator their credential is bad when the platform is simply gone."""
|
|
ok, message = await verify_source_credential(
|
|
platform="pixiv", url="https://www.pixiv.net/users/1", artist_slug="someone",
|
|
config_overrides=None, cookies_path=None, auth_token=None,
|
|
images_root=Path("/nonexistent"),
|
|
)
|
|
assert ok is None
|
|
assert "pixiv" in message
|
|
|
|
|
|
def test_gallery_dl_platforms_are_not_native():
|
|
# The platforms still served by gallery-dl must NOT route to the native
|
|
# ingester — guards an accidental over-broad migration.
|
|
assert uses_native_ingester("hentaifoundry") is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_discord_verify_without_a_token_is_a_rejection_not_a_request():
|
|
"""Discord authenticates by token (milestone 428); with none saved there is
|
|
nothing to send, and saying so beats an HTTP 401 from Discord."""
|
|
ok, message = await verify_source_credential(
|
|
platform="discord", url="https://discord.com/channels/1/2",
|
|
artist_slug="someone", config_overrides=None, cookies_path=None,
|
|
auth_token=None, images_root=Path("/nonexistent"),
|
|
)
|
|
assert ok is False
|
|
assert "token" in message.lower()
|
|
|
|
|
|
def test_unknown_platform_is_not_native():
|
|
assert uses_native_ingester("nonsense") is False
|
|
|
|
|
|
def test_every_native_platform_dispatches_to_an_ingester():
|
|
"""The dispatch table and NATIVE_INGESTER_PLATFORMS have to agree, or a
|
|
platform that routes native raises KeyError mid-download instead of being
|
|
refused up front. Written over the set rather than per-platform so adding
|
|
one to NATIVE_INGESTER_PLATFORMS and forgetting the class fails here.
|
|
|
|
(This replaces the per-platform dispatch tests, one of which was pixiv's;
|
|
it was deleted with pixiv at milestone #406 phase 2.)"""
|
|
for platform in NATIVE_INGESTER_PLATFORMS:
|
|
assert _native_ingester_cls(platform) is not None
|
|
|