feat: Discord on the native core ingester — client, downloader, ledgers, wiring (milestone 428)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is <channel>:<before>. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
"""The native Discord client walks what gallery-dl walked, in its order (#4412).
|
||||
|
||||
No network: a fake session answers by endpoint. What these pin is the part of
|
||||
gallery-dl's behaviour that decides WHICH files exist and WHAT they are called —
|
||||
the channel walk, the file list and its numbering, the name split — because a
|
||||
difference there is a re-download or a missed file at cutover, not a style
|
||||
choice.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services import discord_client as dc
|
||||
from backend.app.services.discord_client import (
|
||||
DiscordAPIError,
|
||||
DiscordAuthError,
|
||||
DiscordClient,
|
||||
firefox_user_agent,
|
||||
message_text,
|
||||
nameext_from_url,
|
||||
parse_source_url,
|
||||
)
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status, body=None, headers=None):
|
||||
self.status_code = status
|
||||
self._body = body
|
||||
self.headers = headers or {}
|
||||
self.content = b"x"
|
||||
|
||||
def json(self):
|
||||
return self._body
|
||||
|
||||
|
||||
class _Session:
|
||||
"""Answers `GET API_ROOT + endpoint` from a dict. A value may be a list of
|
||||
responses, served in order; a messages entry is keyed by (endpoint, before)."""
|
||||
|
||||
def __init__(self, routes):
|
||||
self.routes = routes
|
||||
self.headers = {}
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, params=None, timeout=None):
|
||||
endpoint = url[len(dc.API_ROOT):]
|
||||
self.calls.append((endpoint, dict(params or {})))
|
||||
key = endpoint
|
||||
if endpoint.endswith("/messages"):
|
||||
key = (endpoint, (params or {}).get("before"))
|
||||
elif endpoint.endswith("/threads/search"):
|
||||
key = (endpoint, (params or {}).get("offset"))
|
||||
answer = self.routes.get(key, _Resp(404, {}))
|
||||
if isinstance(answer, list):
|
||||
return answer.pop(0)
|
||||
return answer
|
||||
|
||||
|
||||
def _ok(body):
|
||||
return _Resp(200, body)
|
||||
|
||||
|
||||
def _msg(mid, *, content="", attachments=(), embeds=(), **extra):
|
||||
return {
|
||||
"id": str(mid), "type": 0, "content": content,
|
||||
"timestamp": "2026-09-20T12:00:00.000000+00:00",
|
||||
"author": {"id": "7", "username": "artist"},
|
||||
"attachments": list(attachments), "embeds": list(embeds), **extra,
|
||||
}
|
||||
|
||||
|
||||
def _client(routes, **kw):
|
||||
return DiscordClient("tok", session=_Session(routes), **kw)
|
||||
|
||||
|
||||
def _ids(client, url, cursor=None):
|
||||
return [(m["id"], cur) for m, _meta, cur in client.iter_posts(url, cursor)]
|
||||
|
||||
|
||||
# -- pure helpers --------------------------------------------------------------
|
||||
|
||||
def test_source_urls():
|
||||
assert parse_source_url("https://discord.com/channels/1/2") == ("1", "2")
|
||||
assert parse_source_url("https://discord.com/channels/1") == ("1", None)
|
||||
assert parse_source_url("https://discord.com/channels/1/2/threads/3") == ("1", "3")
|
||||
assert parse_source_url("https://discord.com/channels/@me/5") == (None, "5")
|
||||
assert parse_source_url("discord.com/channels/1/2/") == ("1", "2")
|
||||
|
||||
|
||||
def test_a_message_link_is_not_a_source():
|
||||
with pytest.raises(DiscordAPIError):
|
||||
parse_source_url("https://discord.com/channels/1/2/3")
|
||||
with pytest.raises(DiscordAPIError):
|
||||
parse_source_url("https://example.com/channels/1/2")
|
||||
|
||||
|
||||
def test_name_split_matches_gallery_dl():
|
||||
url = "https://cdn.discordapp.com/attachments/1/2/My%20Pic.final.PNG?ex=a&hm=b"
|
||||
assert nameext_from_url(url) == ("My Pic.final", "png")
|
||||
assert nameext_from_url("https://x/y/noext") == ("noext", "")
|
||||
assert nameext_from_url("https://x/y/a." + "b" * 17) == ("a." + "b" * 17, "")
|
||||
|
||||
|
||||
def test_user_agent_is_gallery_dls_dated_firefox():
|
||||
"""gallery-dl's own comment: "147 on 2026-01-13"."""
|
||||
assert "Firefox/147.0" in firefox_user_agent(date(2026, 1, 13))
|
||||
|
||||
|
||||
def test_message_text_takes_rich_embeds_and_polls():
|
||||
m = _msg(1, content="hello", embeds=[
|
||||
{"type": "rich", "author": {"name": "A"}, "title": "T",
|
||||
"fields": [{"name": "f", "value": "v"}], "footer": {"text": "ft"}},
|
||||
{"type": "image", "title": "not text"},
|
||||
], poll={"question": {"text": "Q?"}, "answers": [{"poll_media": {"text": "yes"}}]})
|
||||
assert message_text(m) == "hello\nA\nT\nf\nv\nft\nQ?\nyes"
|
||||
|
||||
|
||||
def test_files_are_attachments_then_embeds_then_snapshots_numbered_across():
|
||||
m = _msg(9, attachments=[{"url": "https://cdn/a/1.png"}], embeds=[
|
||||
{"type": "video", "video": {"proxy_url": "https://media/v.mp4"},
|
||||
"thumbnail": {"proxy_url": "https://media/t.jpg"}},
|
||||
{"type": "image", "thumbnail": {"proxy_url": "https://media/i.webp"}},
|
||||
{"type": "rich", "image": {"proxy_url": "https://media/rich.png"}},
|
||||
], message_snapshots=[
|
||||
{"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.gif"}],
|
||||
"embeds": []}},
|
||||
{"message": {"type": 7, "attachments": [{"url": "https://cdn/a/join.png"}]}},
|
||||
])
|
||||
items = DiscordClient.extract_media(m)
|
||||
assert [(i.num, i.filename, i.extension, i.kind) for i in items] == [
|
||||
(1, "1", "png", "attachment"),
|
||||
(2, "v", "mp4", "embed"),
|
||||
(3, "i", "webp", "embed"),
|
||||
(4, "fwd", "gif", "attachment"),
|
||||
]
|
||||
assert {i.post_id for i in items} == {"9"}
|
||||
|
||||
|
||||
def test_the_ledger_identity_survives_a_renumbering_edit():
|
||||
"""Removing the first file renumbers the second; its identity must not move."""
|
||||
a = {"id": "100", "url": "https://cdn/a/1.png?ex=1"}
|
||||
b = {"id": "200", "url": "https://cdn/a/2.png?ex=1"}
|
||||
before = DiscordClient.extract_media(_msg(9, attachments=[a, b]))
|
||||
after = DiscordClient.extract_media(_msg(9, attachments=[b]))
|
||||
assert (before[1].num, after[0].num) == (2, 1)
|
||||
assert before[1].media_id == after[0].media_id == "200"
|
||||
|
||||
|
||||
def test_an_embeds_identity_ignores_its_signature():
|
||||
def embed(sig):
|
||||
return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}}
|
||||
|
||||
one = DiscordClient.extract_media(_msg(9, embeds=[embed("a")]))[0].media_id
|
||||
two = DiscordClient.extract_media(_msg(9, embeds=[embed("b")]))[0].media_id
|
||||
assert one == two and len(one) <= 33
|
||||
|
||||
|
||||
def test_post_seams():
|
||||
with_file = _msg(5, attachments=[{"url": "https://cdn/a/1.png"}])
|
||||
assert DiscordClient.post_record_key(with_file) == ("message:5", "5")
|
||||
assert DiscordClient.post_record_key({}) is None
|
||||
|
||||
|
||||
def test_a_text_only_message_is_not_a_post():
|
||||
"""gallery-dl never made one: chat lines would bury the drops."""
|
||||
assert DiscordClient.post_record_key(_msg(6, content="brb")) is None
|
||||
assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20")
|
||||
|
||||
|
||||
# -- the walk --------------------------------------------------------------------
|
||||
|
||||
def test_a_channel_pages_newest_first_and_skips_system_messages():
|
||||
page1 = [_msg(i) for i in range(300, 200, -1)]
|
||||
page1[3]["type"] = 7 # a member-join line: not content
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "art"}]),
|
||||
("/channels/2/messages", None): _ok(page1),
|
||||
("/channels/2/messages", "201"): _ok([_msg(150)]),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||
}
|
||||
got = _ids(_client(routes), "https://discord.com/channels/1/2")
|
||||
assert len(got) == 100 # 99 of page 1 + 1 of page 2
|
||||
assert "297" not in [mid for mid, _ in got]
|
||||
assert got[0] == ("300", "2:")
|
||||
assert got[-1] == ("150", "2:201")
|
||||
|
||||
|
||||
def test_messages_carry_server_and_channel_metadata():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "Studio", "owner_id": "9"}),
|
||||
"/guilds/1/channels": _ok([
|
||||
{"id": "4", "type": 4, "name": "Art"},
|
||||
{"id": "2", "type": 0, "name": "drops", "parent_id": "4"},
|
||||
]),
|
||||
("/channels/2/messages", None): _ok([_msg(10)]),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||
}
|
||||
[(msg, meta, _)] = list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||||
assert msg["_meta"] is meta
|
||||
assert meta["server"] == "Studio" and meta["server_id"] == "1"
|
||||
assert meta["channel"] == "drops" and meta["channel_id"] == "2"
|
||||
assert meta["parent"] == "Art"
|
||||
|
||||
|
||||
def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
"/guilds/1/channels": _ok([
|
||||
{"id": "2", "type": 0, "name": "text"},
|
||||
{"id": "3", "type": 2, "name": "voice"},
|
||||
{"id": "5", "type": 15, "name": "forum"},
|
||||
{"id": "6", "type": 0, "name": "private"},
|
||||
]),
|
||||
("/channels/2/messages", None): _ok([_msg(20)]),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": [
|
||||
{"id": "21", "type": 11, "name": "old", "parent_id": "2", "thread_metadata": {}},
|
||||
{"id": "22", "type": 11, "name": "new", "parent_id": "2", "thread_metadata": {}},
|
||||
]}),
|
||||
("/channels/22/messages", None): _ok([_msg(220)]),
|
||||
("/channels/21/messages", None): _Resp(403, {}), # a private thread
|
||||
("/channels/5/threads/search", 0): _ok({"threads": [
|
||||
{"id": "51", "type": 11, "name": "post", "parent_id": "5", "thread_metadata": {}},
|
||||
]}),
|
||||
("/channels/51/messages", None): _ok([_msg(510)]),
|
||||
("/channels/6/messages", None): _Resp(403, {}),
|
||||
("/channels/6/threads/search", 0): _Resp(403, {}),
|
||||
}
|
||||
got = [mid for mid, _ in _ids(_client(routes), "https://discord.com/channels/1")]
|
||||
assert got == ["20", "220", "510"]
|
||||
|
||||
|
||||
def test_a_resume_cursor_reenters_its_channel_at_its_page():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
"/guilds/1/channels": _ok([
|
||||
{"id": "2", "type": 0, "name": "a"},
|
||||
{"id": "3", "type": 0, "name": "b"},
|
||||
]),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||||
("/channels/3/messages", "77"): _ok([_msg(70)]),
|
||||
}
|
||||
client = _client(routes)
|
||||
assert _ids(client, "https://discord.com/channels/1", "3:77") == [("70", "3:77")]
|
||||
fetched = [c for c in client._session.calls if c[0].endswith("/messages")]
|
||||
assert fetched == [("/channels/3/messages", {"limit": 100, "before": "77"})]
|
||||
|
||||
|
||||
def test_skip_feed_ends_the_channel_not_the_walk():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
"/guilds/1/channels": _ok([
|
||||
{"id": "2", "type": 0, "name": "a"},
|
||||
{"id": "3", "type": 0, "name": "b"},
|
||||
]),
|
||||
("/channels/2/messages", None): _ok([_msg(29), _msg(28)]),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||
("/channels/3/messages", None): _ok([_msg(39)]),
|
||||
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||||
}
|
||||
client = _client(routes)
|
||||
seen = []
|
||||
for msg, _meta, _cur in client.iter_posts("https://discord.com/channels/1"):
|
||||
seen.append(msg["id"])
|
||||
if msg["id"] == "29":
|
||||
client.skip_feed()
|
||||
assert seen == ["29", "39"]
|
||||
|
||||
|
||||
def test_the_named_channel_refusing_the_token_is_an_auth_failure():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "a"}]),
|
||||
("/channels/2/messages", None): _Resp(403, {}),
|
||||
}
|
||||
with pytest.raises(DiscordAuthError):
|
||||
list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||||
|
||||
|
||||
def test_401_is_an_invalid_token():
|
||||
with pytest.raises(DiscordAuthError):
|
||||
list(_client({"/guilds/1": _Resp(401, {})}).iter_posts(
|
||||
"https://discord.com/channels/1"))
|
||||
|
||||
|
||||
def test_no_token_fails_before_any_request():
|
||||
client = DiscordClient(None, session=_Session({}))
|
||||
with pytest.raises(DiscordAuthError):
|
||||
list(client.iter_posts("https://discord.com/channels/1"))
|
||||
assert client._session.calls == []
|
||||
|
||||
|
||||
def test_429_waits_and_retries(monkeypatch):
|
||||
waits = []
|
||||
monkeypatch.setattr(dc.time, "sleep", waits.append)
|
||||
routes = {"/users/@me": [
|
||||
_Resp(429, {}, {"Retry-After": "1.5"}), _ok({"username": "me"}),
|
||||
], "/channels/2": _ok({"id": "2", "type": 0})}
|
||||
ok, msg = _client(routes).verify_auth("https://discord.com/channels/1/2")
|
||||
assert ok is True and "me" in msg
|
||||
assert waits == [1.5]
|
||||
|
||||
|
||||
def test_verify_tells_a_bad_token_from_a_hidden_channel():
|
||||
bad = _client({"/users/@me": _Resp(401, {})})
|
||||
assert bad.verify_auth("https://discord.com/channels/1/2")[0] is False
|
||||
hidden = _client({"/users/@me": _ok({"username": "me"}),
|
||||
"/channels/2": _Resp(403, {})})
|
||||
ok, msg = hidden.verify_auth("https://discord.com/channels/1/2")
|
||||
assert ok is False and "cannot see this channel" in msg
|
||||
assert _client({}).verify_auth("https://discord.com/channels/1/2/3")[0] is None
|
||||
|
||||
|
||||
def test_the_request_profile_is_gallery_dls():
|
||||
client = _client({})
|
||||
h = client._session.headers
|
||||
assert h["Authorization"] == "tok"
|
||||
assert h["Referer"] == "https://discord.com/"
|
||||
assert h["Accept"] == "*/*"
|
||||
assert h["User-Agent"].startswith("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:")
|
||||
|
||||
|
||||
# -- the adapter ---------------------------------------------------------------
|
||||
|
||||
def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path):
|
||||
from backend.app.services.discord_ingester import DiscordIngester, _ledger_key
|
||||
|
||||
ing = DiscordIngester(tmp_path, None, session_factory=None, auth_token="tok")
|
||||
assert ing.client._session.headers["Authorization"] == "tok"
|
||||
# A files-only drop is ordinary on Discord, not a broken parser.
|
||||
assert ing._body_canary is False
|
||||
[media] = DiscordClient.extract_media(
|
||||
_msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
|
||||
)
|
||||
assert _ledger_key(media) == "9:300"
|
||||
@@ -0,0 +1,135 @@
|
||||
"""The native Discord downloader lands files where gallery-dl did (#4414).
|
||||
|
||||
A cutover that names one file differently re-downloads it and imports a
|
||||
duplicate, so these pin the path, the name cleaning and the sidecar pairing
|
||||
against what gallery-dl's config produces — and that the records it writes read
|
||||
back through `parse_sidecar` as the same post the gallery-dl sidecars made.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from backend.app.services.discord_client import DiscordClient
|
||||
from backend.app.services.discord_downloader import (
|
||||
DiscordDownloader,
|
||||
channel_dir,
|
||||
gdl_clean,
|
||||
media_stem,
|
||||
)
|
||||
from backend.app.utils.sidecar import find_sidecar, parse_sidecar
|
||||
|
||||
_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
|
||||
|
||||
class _Resp:
|
||||
status_code = 200
|
||||
headers: dict = {}
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def iter_content(self, chunk_size=None):
|
||||
yield _PNG
|
||||
|
||||
|
||||
class _Media:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, stream=None, timeout=None, headers=None):
|
||||
self.calls.append(url)
|
||||
return _Resp()
|
||||
|
||||
|
||||
def _message(**extra):
|
||||
return {
|
||||
"id": "1234", "type": 0, "timestamp": "2026-09-20T23:30:00.000000+00:00",
|
||||
"content": "new set!", "channel_id": "22",
|
||||
"author": {"id": "7", "username": "artist"},
|
||||
"attachments": [{"url": "https://cdn.discordapp.com/attachments/2/3/Red%3AAlt.PNG?ex=1"}],
|
||||
"embeds": [],
|
||||
"_meta": {"server": "Studio", "server_id": "11", "channel": "drops/nsfw",
|
||||
"channel_id": "22", "is_thread": False},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _downloader(tmp_path, session=None):
|
||||
return DiscordDownloader(tmp_path, validate=False, session=session or _Media())
|
||||
|
||||
|
||||
def test_names_follow_gallery_dls_pattern_and_linux_cleaning(tmp_path):
|
||||
msg = _message()
|
||||
[media] = DiscordClient.extract_media(msg)
|
||||
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord" / "drops_nsfw"
|
||||
# `:` survives: gallery-dl on Linux only replaces `/`.
|
||||
assert media_stem(msg, media) == "20260920_1234_01_Red:Alt"
|
||||
assert gdl_clean("a\x07b/c") == "ab_c"
|
||||
|
||||
|
||||
def test_a_channel_with_no_name_adds_no_directory(tmp_path):
|
||||
msg = _message(_meta={"channel": " "})
|
||||
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord"
|
||||
|
||||
|
||||
def test_a_file_gallery_dl_already_wrote_is_not_fetched_again(tmp_path):
|
||||
msg = _message()
|
||||
media = DiscordClient.extract_media(msg)
|
||||
existing = tmp_path / "art" / "discord" / "drops_nsfw" / "20260920_1234_01_Red:Alt.png"
|
||||
existing.parent.mkdir(parents=True)
|
||||
existing.write_bytes(_PNG)
|
||||
session = _Media()
|
||||
[out] = _downloader(tmp_path, session).download_post(msg, media, "art")
|
||||
assert out.status == "skipped_disk" and out.path == existing
|
||||
assert session.calls == []
|
||||
|
||||
|
||||
def test_a_new_file_gets_a_sidecar_the_importer_pairs_to_its_message(tmp_path):
|
||||
msg = _message()
|
||||
[out] = _downloader(tmp_path).download_post(msg, DiscordClient.extract_media(msg), "art")
|
||||
assert out.status == "downloaded"
|
||||
assert out.path.name == "20260920_1234_01_Red:Alt.png"
|
||||
sidecar = find_sidecar(out.path)
|
||||
assert sidecar is not None
|
||||
data = json.loads(sidecar.read_text())
|
||||
# No `id`/`post_id`: either would outrank message_id as the post id.
|
||||
assert "id" not in data and "post_id" not in data
|
||||
sd = parse_sidecar(data)
|
||||
assert sd.external_post_id == "1234"
|
||||
assert sd.source_url.startswith("https://cdn.discordapp.com/")
|
||||
|
||||
|
||||
def test_a_seen_file_is_skipped_without_a_request(tmp_path):
|
||||
msg = _message()
|
||||
session = _Media()
|
||||
[out] = _downloader(tmp_path, session).download_post(
|
||||
msg, DiscordClient.extract_media(msg), "art", is_seen=lambda m: True,
|
||||
)
|
||||
assert out.status == "skipped_seen" and session.calls == []
|
||||
|
||||
|
||||
def test_the_message_record_reads_back_as_the_gallery_dl_post(tmp_path):
|
||||
rec = _downloader(tmp_path).write_post_record(_message(), "art")
|
||||
assert rec.path.name == "20260920_1234_post.json"
|
||||
assert rec.body_chars == len("new set!")
|
||||
sd = parse_sidecar(json.loads(rec.path.read_text()))
|
||||
assert sd.platform == "discord"
|
||||
assert sd.external_post_id == "1234"
|
||||
assert sd.post_url == "https://discord.com/channels/11/22/1234"
|
||||
assert sd.description == "new set!"
|
||||
assert sd.post_date.isoformat().startswith("2026-09-20T23:30")
|
||||
|
||||
|
||||
def test_the_record_is_not_a_media_sidecar(tmp_path):
|
||||
"""It must not pair with any file of the message."""
|
||||
msg = _message()
|
||||
dl = _downloader(tmp_path)
|
||||
[out] = dl.download_post(msg, DiscordClient.extract_media(msg), "art")
|
||||
rec = dl.write_post_record(msg, "art")
|
||||
assert find_sidecar(out.path) != rec.path
|
||||
|
||||
|
||||
def test_an_empty_re_read_never_blanks_a_stored_body(tmp_path):
|
||||
rec = _downloader(tmp_path).write_post_record(_message(content=""), "art", revisit=True)
|
||||
assert rec.path is None
|
||||
@@ -17,7 +17,7 @@ from backend.app.services.gallery_dl import ErrorType
|
||||
|
||||
|
||||
def test_native_platforms():
|
||||
for platform in ("patreon", "subscribestar"):
|
||||
for platform in ("patreon", "subscribestar", "discord"):
|
||||
assert uses_native_ingester(platform) is True
|
||||
assert platform in NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
@@ -104,8 +104,20 @@ async def test_verifying_a_retired_platform_is_inconclusive_not_rejected():
|
||||
def test_gallery_dl_platforms_are_not_native():
|
||||
# The platforms still served by gallery-dl must NOT route to the native
|
||||
# ingester — guards an accidental over-broad migration.
|
||||
for platform in ("hentaifoundry", "discord"):
|
||||
assert uses_native_ingester(platform) is False
|
||||
assert uses_native_ingester("hentaifoundry") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_verify_without_a_token_is_a_rejection_not_a_request():
|
||||
"""Discord authenticates by token (milestone 428); with none saved there is
|
||||
nothing to send, and saying so beats an HTTP 401 from Discord."""
|
||||
ok, message = await verify_source_credential(
|
||||
platform="discord", url="https://discord.com/channels/1/2",
|
||||
artist_slug="someone", config_overrides=None, cookies_path=None,
|
||||
auth_token=None, images_root=Path("/nonexistent"),
|
||||
)
|
||||
assert ok is False
|
||||
assert "token" in message.lower()
|
||||
|
||||
|
||||
def test_unknown_platform_is_not_native():
|
||||
|
||||
@@ -319,6 +319,64 @@ async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
|
||||
assert client.consumed_posts == 2
|
||||
|
||||
|
||||
class _FeedsClient(_FakeClient):
|
||||
"""A source that is several feeds walked in turn (a Discord server's
|
||||
channels), with the optional `skip_feed` seam (#4413). `feeds` is a list of
|
||||
`pages` lists, one per feed."""
|
||||
|
||||
def __init__(self, feeds):
|
||||
super().__init__([page for pages in feeds for page in pages])
|
||||
self._feeds = feeds
|
||||
self._skip = False
|
||||
self.skips = 0
|
||||
|
||||
def skip_feed(self):
|
||||
self._skip = True
|
||||
self.skips += 1
|
||||
|
||||
def iter_posts(self, campaign_id, cursor=None):
|
||||
for pages in self._feeds:
|
||||
self._skip = False
|
||||
self._pages = pages
|
||||
for item in super().iter_posts(campaign_id, cursor):
|
||||
yield item
|
||||
if self._skip:
|
||||
break
|
||||
|
||||
|
||||
def _seed_seen_media(sync_engine, source_id, media):
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
for m in media:
|
||||
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
|
||||
s.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_quiet_feed_ends_itself_not_the_walk(source_id, sync_engine, tmp_path):
|
||||
"""#4413: a Discord server's first channel is all seen; the tick must still
|
||||
reach the second channel's new file, instead of stopping at the first."""
|
||||
quiet = [_media(f"a{i}", 1) for i in range(1, 5)]
|
||||
_seed_seen_media(sync_engine, source_id, quiet)
|
||||
fresh = _media("b1", 1)
|
||||
client = _FeedsClient([
|
||||
[(None, [(m.post_id, [m]) for m in quiet])],
|
||||
[(None, [("b1", [fresh])])],
|
||||
])
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
result = _ingester(sync_engine, tmp_path, client, downloader).run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
|
||||
)
|
||||
assert result.success is True
|
||||
assert client.skips == 1
|
||||
# The quiet feed stopped after its 2nd seen post; the next feed was walked.
|
||||
assert client.consumed_posts == 3
|
||||
assert downloader.download_calls == 1
|
||||
assert "1 feed(s) caught up" in result.stdout
|
||||
assert "reached end" not in result.stdout
|
||||
|
||||
|
||||
# --- backfill -------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,11 @@ pytestmark = pytest.mark.integration
|
||||
def test_non_serialized_platform_has_no_lock():
|
||||
# gallery-dl platforms aren't capped — they get no lock at all.
|
||||
assert platform_lock("hentaifoundry", ttl_seconds=60) is None
|
||||
assert platform_lock("discord", ttl_seconds=60) is None
|
||||
|
||||
|
||||
def test_discord_is_serialized():
|
||||
# Native since milestone 428, and every source shares one user token.
|
||||
assert platform_lock("discord", ttl_seconds=60) is not None
|
||||
|
||||
|
||||
def test_subscribestar_is_serialized():
|
||||
|
||||
Reference in New Issue
Block a user