bvandeusenandClaude Opus 5.5
29c22afcb7
CI and images / lint (push) Successful in 4s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 21s
CI and images / extension-test (push) Successful in 20s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Successful in 2m24s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 7s
CI and images / build-web (push) Successful in 1m42s
CI and images / smoke-web (push) Successful in 59s
CI and images / promote (push) Successful in 2s
- A message with no image or video attachment is chat: extract_media returns nothing for it, so nothing downloads and no post record is written. A message that passes keeps every file, numbered as gallery-dl numbers them. - `discord_authors` in a source's config limits the walk to those posters (id, username or display name); the edit dialog has a field for it and no longer drops config keys it has no field for. - source.display_name (migration 0115), refreshed by every Discord walk, shows on Subscriptions in place of the two-id URL; a Discord post card names its channel from the record's `channel`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
427 lines
17 KiB
Python
427 lines
17 KiB
Python
"""The native Discord client walks what gallery-dl walked, in its order (#4412).
|
||
|
||
No network: a fake session answers by endpoint. What these pin is the part of
|
||
gallery-dl's behaviour that decides WHICH files exist and WHAT they are called —
|
||
the channel walk, the file list and its numbering, the name split — because a
|
||
difference there is a re-download or a missed file at cutover, not a style
|
||
choice.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from backend.app.services import discord_client as dc
|
||
from backend.app.services.discord_client import (
|
||
DiscordAPIError,
|
||
DiscordAuthError,
|
||
DiscordClient,
|
||
firefox_user_agent,
|
||
message_text,
|
||
nameext_from_url,
|
||
parse_source_url,
|
||
)
|
||
|
||
|
||
class _Resp:
|
||
def __init__(self, status, body=None, headers=None):
|
||
self.status_code = status
|
||
self._body = body
|
||
self.headers = headers or {}
|
||
self.content = b"x"
|
||
|
||
def json(self):
|
||
return self._body
|
||
|
||
|
||
class _Session:
|
||
"""Answers `GET API_ROOT + endpoint` from a dict. A value may be a list of
|
||
responses, served in order; a messages entry is keyed by (endpoint, before)."""
|
||
|
||
def __init__(self, routes):
|
||
self.routes = routes
|
||
self.headers = {}
|
||
self.calls = []
|
||
|
||
def get(self, url, params=None, timeout=None):
|
||
endpoint = url[len(dc.API_ROOT):]
|
||
self.calls.append((endpoint, dict(params or {})))
|
||
key = endpoint
|
||
if endpoint.endswith("/messages"):
|
||
key = (endpoint, (params or {}).get("before"))
|
||
elif endpoint.endswith("/threads/search"):
|
||
key = (endpoint, (params or {}).get("offset"))
|
||
answer = self.routes.get(key, _Resp(404, {}))
|
||
if isinstance(answer, list):
|
||
return answer.pop(0)
|
||
return answer
|
||
|
||
|
||
def _ok(body):
|
||
return _Resp(200, body)
|
||
|
||
|
||
def _msg(mid, *, content="", attachments=(), embeds=(), **extra):
|
||
return {
|
||
"id": str(mid), "type": 0, "content": content,
|
||
"timestamp": "2026-09-20T12:00:00.000000+00:00",
|
||
"author": {"id": "7", "username": "artist"},
|
||
"attachments": list(attachments), "embeds": list(embeds), **extra,
|
||
}
|
||
|
||
|
||
def _client(routes, **kw):
|
||
return DiscordClient("tok", session=_Session(routes), **kw)
|
||
|
||
|
||
def _ids(client, url, cursor=None):
|
||
return [(m["id"], cur) for m, _meta, cur in client.iter_posts(url, cursor)]
|
||
|
||
|
||
# -- pure helpers --------------------------------------------------------------
|
||
|
||
def test_source_urls():
|
||
assert parse_source_url("https://discord.com/channels/1/2") == ("1", "2")
|
||
assert parse_source_url("https://discord.com/channels/1") == ("1", None)
|
||
assert parse_source_url("https://discord.com/channels/1/2/threads/3") == ("1", "3")
|
||
assert parse_source_url("https://discord.com/channels/@me/5") == (None, "5")
|
||
assert parse_source_url("discord.com/channels/1/2/") == ("1", "2")
|
||
|
||
|
||
def test_a_message_link_is_not_a_source():
|
||
with pytest.raises(DiscordAPIError):
|
||
parse_source_url("https://discord.com/channels/1/2/3")
|
||
with pytest.raises(DiscordAPIError):
|
||
parse_source_url("https://example.com/channels/1/2")
|
||
|
||
|
||
def test_name_split_matches_gallery_dl():
|
||
url = "https://cdn.discordapp.com/attachments/1/2/My%20Pic.final.PNG?ex=a&hm=b"
|
||
assert nameext_from_url(url) == ("My Pic.final", "png")
|
||
assert nameext_from_url("https://x/y/noext") == ("noext", "")
|
||
assert nameext_from_url("https://x/y/a." + "b" * 17) == ("a." + "b" * 17, "")
|
||
|
||
|
||
def test_user_agent_is_gallery_dls_dated_firefox():
|
||
"""gallery-dl's own comment: "147 on 2026-01-13"."""
|
||
assert "Firefox/147.0" in firefox_user_agent(date(2026, 1, 13))
|
||
|
||
|
||
def test_message_text_takes_rich_embeds_and_polls():
|
||
m = _msg(1, content="hello", embeds=[
|
||
{"type": "rich", "author": {"name": "A"}, "title": "T",
|
||
"fields": [{"name": "f", "value": "v"}], "footer": {"text": "ft"}},
|
||
{"type": "image", "title": "not text"},
|
||
], poll={"question": {"text": "Q?"}, "answers": [{"poll_media": {"text": "yes"}}]})
|
||
assert message_text(m) == "hello\nA\nT\nf\nv\nft\nQ?\nyes"
|
||
|
||
|
||
def test_files_are_attachments_then_embeds_then_snapshots_numbered_across():
|
||
m = _msg(9, attachments=[{"url": "https://cdn/a/1.png"}], embeds=[
|
||
{"type": "video", "video": {"proxy_url": "https://media/v.mp4"},
|
||
"thumbnail": {"proxy_url": "https://media/t.jpg"}},
|
||
{"type": "image", "thumbnail": {"proxy_url": "https://media/i.webp"}},
|
||
{"type": "rich", "image": {"proxy_url": "https://media/rich.png"}},
|
||
], message_snapshots=[
|
||
{"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.gif"}],
|
||
"embeds": []}},
|
||
{"message": {"type": 7, "attachments": [{"url": "https://cdn/a/join.png"}]}},
|
||
])
|
||
items = DiscordClient.extract_media(m)
|
||
assert [(i.num, i.filename, i.extension, i.kind) for i in items] == [
|
||
(1, "1", "png", "attachment"),
|
||
(2, "v", "mp4", "embed"),
|
||
(3, "i", "webp", "embed"),
|
||
(4, "fwd", "gif", "attachment"),
|
||
]
|
||
assert {i.post_id for i in items} == {"9"}
|
||
|
||
|
||
def test_the_ledger_identity_survives_a_renumbering_edit():
|
||
"""Removing the first file renumbers the second; its identity must not move."""
|
||
a = {"id": "100", "url": "https://cdn/a/1.png?ex=1"}
|
||
b = {"id": "200", "url": "https://cdn/a/2.png?ex=1"}
|
||
before = DiscordClient.extract_media(_msg(9, attachments=[a, b]))
|
||
after = DiscordClient.extract_media(_msg(9, attachments=[b]))
|
||
assert (before[1].num, after[0].num) == (2, 1)
|
||
assert before[1].media_id == after[0].media_id == "200"
|
||
|
||
|
||
def test_an_embeds_identity_ignores_its_signature():
|
||
def embed(sig):
|
||
return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}}
|
||
|
||
art = [{"url": "https://cdn/a/1.png"}]
|
||
one = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("a")]))[1]
|
||
two = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("b")]))[1]
|
||
assert one.kind == two.kind == "embed"
|
||
assert one.media_id == two.media_id and len(one.media_id) <= 33
|
||
|
||
|
||
def test_post_seams():
|
||
with_file = _msg(5, attachments=[{"url": "https://cdn/a/1.png"}])
|
||
assert DiscordClient.post_record_key(with_file) == ("message:5", "5")
|
||
assert DiscordClient.post_record_key({}) is None
|
||
|
||
|
||
def test_a_text_only_message_is_not_a_post():
|
||
"""Chat lines would bury the drops."""
|
||
assert DiscordClient.post_record_key(_msg(6, content="brb")) is None
|
||
assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20")
|
||
|
||
|
||
def test_a_message_without_an_attached_image_is_chat():
|
||
"""Operator 2026-09-28: only content with an image attached. A lone
|
||
archive, a link preview or a Tenor GIF takes nothing and records nothing."""
|
||
chat = [
|
||
_msg(1, content="the stash", attachments=[
|
||
{"url": "https://cdn/a/Links_Stash.rar", "content_type": "application/x-rar"},
|
||
]),
|
||
_msg(2, content="look", embeds=[
|
||
{"type": "image", "image": {"proxy_url": "https://media/p/x.png"}},
|
||
]),
|
||
_msg(3, content="lol", embeds=[
|
||
{"type": "gifv", "video": {"proxy_url": "https://media/p/t.mp4"}},
|
||
]),
|
||
_msg(4, content="wip", attachments=[{"url": "https://cdn/a/wip.psd"}]),
|
||
]
|
||
for m in chat:
|
||
assert DiscordClient.extract_media(m) == []
|
||
assert DiscordClient.post_record_key(m) is None
|
||
|
||
|
||
def test_an_attached_image_takes_every_file_numbered_as_gallery_dl_does():
|
||
"""The gate decides WHETHER a message is taken, never which of its files:
|
||
the rar beside the image keeps its number, so on-disk names still match."""
|
||
m = _msg(7, attachments=[
|
||
{"url": "https://cdn/a/pack.rar"},
|
||
{"url": "https://cdn/a/noext", "content_type": "image/png"},
|
||
])
|
||
items = DiscordClient.extract_media(m)
|
||
assert [(i.num, i.filename) for i in items] == [(1, "pack"), (2, "noext")]
|
||
assert DiscordClient.post_record_key(m) == ("message:7", "7")
|
||
video = _msg(8, attachments=[{"url": "https://cdn/a/clip.MP4?ex=1"}])
|
||
assert len(DiscordClient.extract_media(video)) == 1
|
||
|
||
|
||
def test_a_forwarded_image_counts_as_attached():
|
||
m = _msg(9, message_snapshots=[
|
||
{"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.png"}]}},
|
||
])
|
||
assert DiscordClient.post_record_key(m) == ("message:9", "9")
|
||
|
||
|
||
# -- the walk --------------------------------------------------------------------
|
||
|
||
def test_a_channel_pages_newest_first_and_skips_system_messages():
|
||
page1 = [_msg(i) for i in range(300, 200, -1)]
|
||
page1[3]["type"] = 7 # a member-join line: not content
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "art"}]),
|
||
("/channels/2/messages", None): _ok(page1),
|
||
("/channels/2/messages", "201"): _ok([_msg(150)]),
|
||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||
}
|
||
got = _ids(_client(routes), "https://discord.com/channels/1/2")
|
||
assert len(got) == 100 # 99 of page 1 + 1 of page 2
|
||
assert "297" not in [mid for mid, _ in got]
|
||
assert got[0] == ("300", "2:")
|
||
assert got[-1] == ("150", "2:201")
|
||
|
||
|
||
def test_messages_carry_server_and_channel_metadata():
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "Studio", "owner_id": "9"}),
|
||
"/guilds/1/channels": _ok([
|
||
{"id": "4", "type": 4, "name": "Art"},
|
||
{"id": "2", "type": 0, "name": "drops", "parent_id": "4"},
|
||
]),
|
||
("/channels/2/messages", None): _ok([_msg(10)]),
|
||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||
}
|
||
[(msg, meta, _)] = list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||
assert msg["_meta"] is meta
|
||
assert meta["server"] == "Studio" and meta["server_id"] == "1"
|
||
assert meta["channel"] == "drops" and meta["channel_id"] == "2"
|
||
assert meta["parent"] == "Art"
|
||
|
||
|
||
def _one_channel(messages):
|
||
return {
|
||
"/guilds/1": _ok({"id": "1", "name": "Todding's Server"}),
|
||
"/guilds/1/channels": _ok([
|
||
{"id": "4", "type": 4, "name": "Todding"},
|
||
{"id": "2", "type": 0, "name": "banana-land", "parent_id": "4"},
|
||
]),
|
||
("/channels/2/messages", None): _ok(messages),
|
||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||
}
|
||
|
||
|
||
def test_only_from_takes_the_creators_messages_and_no_one_elses():
|
||
"""Operator 2026-09-28: in the creator's server, another member posting a
|
||
meme is not the creator's art. Matched by id, username or display name."""
|
||
creator = {"id": "7", "username": "todding", "global_name": "Todding"}
|
||
other = {"id": "8", "username": "jakeboii", "global_name": "Jake Boii"}
|
||
msgs = [_msg(3, author=creator), _msg(2, author=other), _msg(1, author=creator)]
|
||
url = "https://discord.com/channels/1/2"
|
||
|
||
everyone = _client(_one_channel([dict(m) for m in msgs]))
|
||
assert [mid for mid, _ in _ids(everyone, url)] == ["3", "2", "1"]
|
||
for who in (["Todding"], ["TODDING "], ["7"]):
|
||
client = _client(_one_channel([dict(m) for m in msgs]))
|
||
client.only_from(who)
|
||
assert [mid for mid, _ in _ids(client, url)] == ["3", "1"]
|
||
client = _client(_one_channel([dict(m) for m in msgs]))
|
||
client.only_from([])
|
||
assert len(_ids(client, url)) == 3
|
||
|
||
|
||
def test_the_source_label_names_the_server_and_channel_the_walk_read():
|
||
client = _client(_one_channel([]))
|
||
url = "https://discord.com/channels/1/2"
|
||
assert client.source_label(url) is None # nothing walked yet
|
||
list(client.iter_posts(url))
|
||
assert client.source_label(url) == "Todding's Server · #banana-land"
|
||
assert client.source_label("https://discord.com/channels/1") == "Todding's Server"
|
||
client._channels["9"] = {"channel": "wip", "is_thread": True, "parent": "banana-land"}
|
||
assert client.source_label("https://discord.com/channels/1/9") == (
|
||
"Todding's Server · #banana-land › wip"
|
||
)
|
||
|
||
|
||
def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private():
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||
"/guilds/1/channels": _ok([
|
||
{"id": "2", "type": 0, "name": "text"},
|
||
{"id": "3", "type": 2, "name": "voice"},
|
||
{"id": "5", "type": 15, "name": "forum"},
|
||
{"id": "6", "type": 0, "name": "private"},
|
||
]),
|
||
("/channels/2/messages", None): _ok([_msg(20)]),
|
||
("/channels/2/threads/search", 0): _ok({"threads": [
|
||
{"id": "21", "type": 11, "name": "old", "parent_id": "2", "thread_metadata": {}},
|
||
{"id": "22", "type": 11, "name": "new", "parent_id": "2", "thread_metadata": {}},
|
||
]}),
|
||
("/channels/22/messages", None): _ok([_msg(220)]),
|
||
("/channels/21/messages", None): _Resp(403, {}), # a private thread
|
||
("/channels/5/threads/search", 0): _ok({"threads": [
|
||
{"id": "51", "type": 11, "name": "post", "parent_id": "5", "thread_metadata": {}},
|
||
]}),
|
||
("/channels/51/messages", None): _ok([_msg(510)]),
|
||
("/channels/6/messages", None): _Resp(403, {}),
|
||
("/channels/6/threads/search", 0): _Resp(403, {}),
|
||
}
|
||
got = [mid for mid, _ in _ids(_client(routes), "https://discord.com/channels/1")]
|
||
assert got == ["20", "220", "510"]
|
||
|
||
|
||
def test_a_resume_cursor_reenters_its_channel_at_its_page():
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||
"/guilds/1/channels": _ok([
|
||
{"id": "2", "type": 0, "name": "a"},
|
||
{"id": "3", "type": 0, "name": "b"},
|
||
]),
|
||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||
("/channels/3/messages", "77"): _ok([_msg(70)]),
|
||
}
|
||
client = _client(routes)
|
||
assert _ids(client, "https://discord.com/channels/1", "3:77") == [("70", "3:77")]
|
||
fetched = [c for c in client._session.calls if c[0].endswith("/messages")]
|
||
assert fetched == [("/channels/3/messages", {"limit": 100, "before": "77"})]
|
||
|
||
|
||
def test_skip_feed_ends_the_channel_not_the_walk():
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||
"/guilds/1/channels": _ok([
|
||
{"id": "2", "type": 0, "name": "a"},
|
||
{"id": "3", "type": 0, "name": "b"},
|
||
]),
|
||
("/channels/2/messages", None): _ok([_msg(29), _msg(28)]),
|
||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||
("/channels/3/messages", None): _ok([_msg(39)]),
|
||
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||
}
|
||
client = _client(routes)
|
||
seen = []
|
||
for msg, _meta, _cur in client.iter_posts("https://discord.com/channels/1"):
|
||
seen.append(msg["id"])
|
||
if msg["id"] == "29":
|
||
client.skip_feed()
|
||
assert seen == ["29", "39"]
|
||
|
||
|
||
def test_the_named_channel_refusing_the_token_is_an_auth_failure():
|
||
routes = {
|
||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "a"}]),
|
||
("/channels/2/messages", None): _Resp(403, {}),
|
||
}
|
||
with pytest.raises(DiscordAuthError):
|
||
list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||
|
||
|
||
def test_401_is_an_invalid_token():
|
||
with pytest.raises(DiscordAuthError):
|
||
list(_client({"/guilds/1": _Resp(401, {})}).iter_posts(
|
||
"https://discord.com/channels/1"))
|
||
|
||
|
||
def test_no_token_fails_before_any_request():
|
||
client = DiscordClient(None, session=_Session({}))
|
||
with pytest.raises(DiscordAuthError):
|
||
list(client.iter_posts("https://discord.com/channels/1"))
|
||
assert client._session.calls == []
|
||
|
||
|
||
def test_429_waits_and_retries(monkeypatch):
|
||
waits = []
|
||
monkeypatch.setattr(dc.time, "sleep", waits.append)
|
||
routes = {"/users/@me": [
|
||
_Resp(429, {}, {"Retry-After": "1.5"}), _ok({"username": "me"}),
|
||
], "/channels/2": _ok({"id": "2", "type": 0})}
|
||
ok, msg = _client(routes).verify_auth("https://discord.com/channels/1/2")
|
||
assert ok is True and "me" in msg
|
||
assert waits == [1.5]
|
||
|
||
|
||
def test_verify_tells_a_bad_token_from_a_hidden_channel():
|
||
bad = _client({"/users/@me": _Resp(401, {})})
|
||
assert bad.verify_auth("https://discord.com/channels/1/2")[0] is False
|
||
hidden = _client({"/users/@me": _ok({"username": "me"}),
|
||
"/channels/2": _Resp(403, {})})
|
||
ok, msg = hidden.verify_auth("https://discord.com/channels/1/2")
|
||
assert ok is False and "cannot see this channel" in msg
|
||
assert _client({}).verify_auth("https://discord.com/channels/1/2/3")[0] is None
|
||
|
||
|
||
def test_the_request_profile_is_gallery_dls():
|
||
client = _client({})
|
||
h = client._session.headers
|
||
assert h["Authorization"] == "tok"
|
||
assert h["Referer"] == "https://discord.com/"
|
||
assert h["Accept"] == "*/*"
|
||
assert h["User-Agent"].startswith("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:")
|
||
|
||
|
||
# -- the adapter ---------------------------------------------------------------
|
||
|
||
def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path):
|
||
from backend.app.services.discord_ingester import DiscordIngester, _ledger_key
|
||
|
||
ing = DiscordIngester(tmp_path, None, session_factory=None, auth_token="tok")
|
||
assert ing.client._session.headers["Authorization"] == "tok"
|
||
# A files-only drop is ordinary on Discord, not a broken parser.
|
||
assert ing._body_canary is False
|
||
[media] = DiscordClient.extract_media(
|
||
_msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
|
||
)
|
||
assert _ledger_key(media) == "9:300"
|