bvandeusenandClaude Opus 5.5
84e5448941
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is <channel>:<before>. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
136 lines
4.8 KiB
Python
136 lines
4.8 KiB
Python
"""The native Discord downloader lands files where gallery-dl did (#4414).
|
|
|
|
A cutover that names one file differently re-downloads it and imports a
|
|
duplicate, so these pin the path, the name cleaning and the sidecar pairing
|
|
against what gallery-dl's config produces — and that the records it writes read
|
|
back through `parse_sidecar` as the same post the gallery-dl sidecars made.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from backend.app.services.discord_client import DiscordClient
|
|
from backend.app.services.discord_downloader import (
|
|
DiscordDownloader,
|
|
channel_dir,
|
|
gdl_clean,
|
|
media_stem,
|
|
)
|
|
from backend.app.utils.sidecar import find_sidecar, parse_sidecar
|
|
|
|
_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
|
|
|
|
|
class _Resp:
|
|
status_code = 200
|
|
headers: dict = {}
|
|
|
|
def raise_for_status(self):
|
|
pass
|
|
|
|
def iter_content(self, chunk_size=None):
|
|
yield _PNG
|
|
|
|
|
|
class _Media:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def get(self, url, stream=None, timeout=None, headers=None):
|
|
self.calls.append(url)
|
|
return _Resp()
|
|
|
|
|
|
def _message(**extra):
|
|
return {
|
|
"id": "1234", "type": 0, "timestamp": "2026-09-20T23:30:00.000000+00:00",
|
|
"content": "new set!", "channel_id": "22",
|
|
"author": {"id": "7", "username": "artist"},
|
|
"attachments": [{"url": "https://cdn.discordapp.com/attachments/2/3/Red%3AAlt.PNG?ex=1"}],
|
|
"embeds": [],
|
|
"_meta": {"server": "Studio", "server_id": "11", "channel": "drops/nsfw",
|
|
"channel_id": "22", "is_thread": False},
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _downloader(tmp_path, session=None):
|
|
return DiscordDownloader(tmp_path, validate=False, session=session or _Media())
|
|
|
|
|
|
def test_names_follow_gallery_dls_pattern_and_linux_cleaning(tmp_path):
|
|
msg = _message()
|
|
[media] = DiscordClient.extract_media(msg)
|
|
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord" / "drops_nsfw"
|
|
# `:` survives: gallery-dl on Linux only replaces `/`.
|
|
assert media_stem(msg, media) == "20260920_1234_01_Red:Alt"
|
|
assert gdl_clean("a\x07b/c") == "ab_c"
|
|
|
|
|
|
def test_a_channel_with_no_name_adds_no_directory(tmp_path):
|
|
msg = _message(_meta={"channel": " "})
|
|
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord"
|
|
|
|
|
|
def test_a_file_gallery_dl_already_wrote_is_not_fetched_again(tmp_path):
|
|
msg = _message()
|
|
media = DiscordClient.extract_media(msg)
|
|
existing = tmp_path / "art" / "discord" / "drops_nsfw" / "20260920_1234_01_Red:Alt.png"
|
|
existing.parent.mkdir(parents=True)
|
|
existing.write_bytes(_PNG)
|
|
session = _Media()
|
|
[out] = _downloader(tmp_path, session).download_post(msg, media, "art")
|
|
assert out.status == "skipped_disk" and out.path == existing
|
|
assert session.calls == []
|
|
|
|
|
|
def test_a_new_file_gets_a_sidecar_the_importer_pairs_to_its_message(tmp_path):
|
|
msg = _message()
|
|
[out] = _downloader(tmp_path).download_post(msg, DiscordClient.extract_media(msg), "art")
|
|
assert out.status == "downloaded"
|
|
assert out.path.name == "20260920_1234_01_Red:Alt.png"
|
|
sidecar = find_sidecar(out.path)
|
|
assert sidecar is not None
|
|
data = json.loads(sidecar.read_text())
|
|
# No `id`/`post_id`: either would outrank message_id as the post id.
|
|
assert "id" not in data and "post_id" not in data
|
|
sd = parse_sidecar(data)
|
|
assert sd.external_post_id == "1234"
|
|
assert sd.source_url.startswith("https://cdn.discordapp.com/")
|
|
|
|
|
|
def test_a_seen_file_is_skipped_without_a_request(tmp_path):
|
|
msg = _message()
|
|
session = _Media()
|
|
[out] = _downloader(tmp_path, session).download_post(
|
|
msg, DiscordClient.extract_media(msg), "art", is_seen=lambda m: True,
|
|
)
|
|
assert out.status == "skipped_seen" and session.calls == []
|
|
|
|
|
|
def test_the_message_record_reads_back_as_the_gallery_dl_post(tmp_path):
|
|
rec = _downloader(tmp_path).write_post_record(_message(), "art")
|
|
assert rec.path.name == "20260920_1234_post.json"
|
|
assert rec.body_chars == len("new set!")
|
|
sd = parse_sidecar(json.loads(rec.path.read_text()))
|
|
assert sd.platform == "discord"
|
|
assert sd.external_post_id == "1234"
|
|
assert sd.post_url == "https://discord.com/channels/11/22/1234"
|
|
assert sd.description == "new set!"
|
|
assert sd.post_date.isoformat().startswith("2026-09-20T23:30")
|
|
|
|
|
|
def test_the_record_is_not_a_media_sidecar(tmp_path):
|
|
"""It must not pair with any file of the message."""
|
|
msg = _message()
|
|
dl = _downloader(tmp_path)
|
|
[out] = dl.download_post(msg, DiscordClient.extract_media(msg), "art")
|
|
rec = dl.write_post_record(msg, "art")
|
|
assert find_sidecar(out.path) != rec.path
|
|
|
|
|
|
def test_an_empty_re_read_never_blanks_a_stored_body(tmp_path):
|
|
rec = _downloader(tmp_path).write_post_record(_message(content=""), "art", revisit=True)
|
|
assert rec.path is None
|