feat: Discord on the native core ingester — client, downloader, ledgers, wiring (milestone 428)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m27s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 1m48s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 1s
Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is <channel>:<before>. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
@@ -0,0 +1,75 @@
|
|||||||
|
"""Discord native ingester ledgers — seen and dead-letter, per source.
|
||||||
|
|
||||||
|
Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
|
||||||
|
keeps its memory of what a source has already fetched in these two tables
|
||||||
|
instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
|
||||||
|
|
||||||
|
Revision ID: 0111
|
||||||
|
Revises: 0110
|
||||||
|
Create Date: 2026-09-24
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0111"
|
||||||
|
down_revision = "0110"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table(
|
||||||
|
"discord_seen_media",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("post_id", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"], ["source.id"],
|
||||||
|
name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
|
||||||
|
sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"discord_failed_media",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||||
|
sa.Column("last_error", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"first_failed_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"last_failed_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"], ["source.id"],
|
||||||
|
name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"source_id", "filehash", name="uq_discord_failed_media_source_id",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
|
||||||
|
op.drop_table("discord_failed_media")
|
||||||
|
op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
|
||||||
|
op.drop_table("discord_seen_media")
|
||||||
@@ -8,6 +8,8 @@ from .backup_run import BackupRun
|
|||||||
from .base import Base
|
from .base import Base
|
||||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||||
from .credential import Credential
|
from .credential import Credential
|
||||||
|
from .discord_failed_media import DiscordFailedMedia
|
||||||
|
from .discord_seen_media import DiscordSeenMedia
|
||||||
from .download_event import DownloadEvent
|
from .download_event import DownloadEvent
|
||||||
from .external_link import ExternalLink
|
from .external_link import ExternalLink
|
||||||
from .gpu_job import GpuJob
|
from .gpu_job import GpuJob
|
||||||
@@ -56,6 +58,8 @@ __all__ = [
|
|||||||
"BackupRun",
|
"BackupRun",
|
||||||
"Source",
|
"Source",
|
||||||
"Credential",
|
"Credential",
|
||||||
|
"DiscordFailedMedia",
|
||||||
|
"DiscordSeenMedia",
|
||||||
"PatreonFailedMedia",
|
"PatreonFailedMedia",
|
||||||
"PatreonSeenMedia",
|
"PatreonSeenMedia",
|
||||||
"SubscribeStarFailedMedia",
|
"SubscribeStarFailedMedia",
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that
|
||||||
|
keep failing to download or validate.
|
||||||
|
|
||||||
|
Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter
|
||||||
|
threshold a routine walk skips the file (recovery still retries it); a later
|
||||||
|
clean download clears the row. `filehash` is the seen-ledger's key.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import DateTime
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordFailedMedia(Base):
|
||||||
|
__tablename__ = "discord_failed_media"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||||
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
first_failed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
last_failed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""DiscordSeenMedia — per-source ledger of Discord files already downloaded.
|
||||||
|
|
||||||
|
Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone
|
||||||
|
428). `filehash` holds the ingester's per-file key, `<message_id>:<media_id>`:
|
||||||
|
the attachment id, or for an embed a hash of its URL path. Not the file's
|
||||||
|
position in the message — an edit that removes a file renumbers the rest
|
||||||
|
(see `discord_client.MediaItem`). The message record's own gate is the
|
||||||
|
synthetic `message:<id>` key in the same column.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import DateTime
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordSeenMedia(Base):
|
||||||
|
__tablename__ = "discord_seen_media"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -0,0 +1,544 @@
|
|||||||
|
"""Native Discord read client — the Discord counterpart to subscribestar_client.
|
||||||
|
|
||||||
|
Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
|
||||||
|
known-working base), adapted to the native core's client contract
|
||||||
|
(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
|
||||||
|
post-first `post_record_key` and the `post_meta` date the revisit window reads.
|
||||||
|
|
||||||
|
What is mirrored exactly, because drift in any of it changes what we fetch or
|
||||||
|
where it lands on disk:
|
||||||
|
|
||||||
|
- API v10, `Authorization: <user token>` (a USER token, not a bot token).
|
||||||
|
- gallery-dl's request profile: its date-derived Firefox User-Agent,
|
||||||
|
`Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
|
||||||
|
- `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
|
||||||
|
stopping on a short page. Message types {0, 19, 21} only.
|
||||||
|
- The walk: a text/news channel's own messages then its threads, a forum's
|
||||||
|
threads, a category's children, a server's text/news/forum channels.
|
||||||
|
- Files: attachments, then embeds of type image/gifv/video (FC configures
|
||||||
|
`embeds: all`, which for files is the same three plus rich/link embeds that
|
||||||
|
carry an image), then forwarded `message_snapshots`, numbered from 1 across
|
||||||
|
the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
|
||||||
|
- Text: `content`, rich-embed author/title/description/fields/footer, poll.
|
||||||
|
|
||||||
|
Two deliberate departures, both about the walk order, neither about content:
|
||||||
|
|
||||||
|
- Threads are walked newest-CREATED first (by id), not by last-message time.
|
||||||
|
A backfill resumes from a checkpointed channel; last-message order shifts
|
||||||
|
between chunks whenever someone posts, which can move an unwalked thread
|
||||||
|
above the resume point and skip it. Creation order only ever grows at the
|
||||||
|
front, where the next tick finds it.
|
||||||
|
- A 403 on a thread or a nested channel skips that feed instead of failing
|
||||||
|
the walk. gallery-dl skips only nested channels; one private thread the
|
||||||
|
token cannot read would otherwise stop every channel after it.
|
||||||
|
|
||||||
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .native_ingest_common import (
|
||||||
|
NativeAuthError,
|
||||||
|
NativeDriftError,
|
||||||
|
NativeIngestError,
|
||||||
|
retry_after_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
API_ROOT = "https://discord.com/api/v10"
|
||||||
|
_ROOT = "https://discord.com"
|
||||||
|
|
||||||
|
_TIMEOUT_SECONDS = 60.0
|
||||||
|
_MESSAGES_BATCH = 100
|
||||||
|
_THREADS_BATCH = 25
|
||||||
|
# gallery-dl retries a 429 up to its default 4 retries, waiting
|
||||||
|
# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
|
||||||
|
# it is honoured when present; 60s is the fallback and the cap.
|
||||||
|
_MAX_429_RETRIES = 4
|
||||||
|
_429_WAIT_SECONDS = 60.0
|
||||||
|
|
||||||
|
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||||
|
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||||
|
MESSAGE_TYPES = frozenset({0, 19, 21})
|
||||||
|
# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
|
||||||
|
_TEXT = frozenset({0, 5}) # text, announcement: messages + threads
|
||||||
|
_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only
|
||||||
|
_FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||||
|
_CATEGORY = 4
|
||||||
|
_SERVER_WALK = _TEXT | _FORUM
|
||||||
|
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||||
|
|
||||||
|
_URL_RE = re.compile(
|
||||||
|
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||||
|
r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordAPIError(NativeIngestError):
|
||||||
|
"""Base for native Discord client failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordAuthError(DiscordAPIError, NativeAuthError):
|
||||||
|
"""401 (the token is invalid or expired) or a 403 on the channel the
|
||||||
|
source names. The fix is a new token, not a new client."""
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordDriftError(DiscordAPIError, NativeDriftError):
|
||||||
|
"""A response did not have the shape the walk depends on."""
|
||||||
|
|
||||||
|
|
||||||
|
def firefox_user_agent(today: date | None = None) -> str:
|
||||||
|
"""gallery-dl's default User-Agent: a Firefox whose version advances every
|
||||||
|
four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
|
||||||
|
the profile keeps matching the gallery-dl this replaced."""
|
||||||
|
ver = ((today or date.today()).toordinal() - 735_513) // 28
|
||||||
|
return (
|
||||||
|
f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
|
||||||
|
f"Gecko/20100101 Firefox/{ver}.0"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def nameext_from_url(url: str) -> tuple[str, str]:
|
||||||
|
"""gallery-dl's `text.nameext_from_url`: the URL's last path segment,
|
||||||
|
unquoted, split at the last dot when the extension is at most 16 chars
|
||||||
|
(lowercased); otherwise the whole name and no extension."""
|
||||||
|
filename = unquote(url.partition("?")[0].rpartition("/")[2])
|
||||||
|
name, _, ext = filename.rpartition(".")
|
||||||
|
if name and len(ext) <= 16:
|
||||||
|
return name, ext.lower()
|
||||||
|
return filename, ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_source_url(url: str) -> tuple[str | None, str | None]:
|
||||||
|
"""`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
|
||||||
|
is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
|
||||||
|
DiscordAPIError for anything else, including a link to a single message —
|
||||||
|
a message is not something a source can subscribe to."""
|
||||||
|
m = _URL_RE.match((url or "").strip())
|
||||||
|
if not m or (m.group("rest") or "").strip("/"):
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Not a Discord channel or server link: {url!r} "
|
||||||
|
"(expected https://discord.com/channels/<server>[/<channel>])"
|
||||||
|
)
|
||||||
|
server = m.group("server")
|
||||||
|
channel = m.group("channel")
|
||||||
|
if server == "@me":
|
||||||
|
if not channel:
|
||||||
|
raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
|
||||||
|
return None, channel
|
||||||
|
return server, channel
|
||||||
|
|
||||||
|
|
||||||
|
def message_text(message: dict) -> str:
|
||||||
|
"""gallery-dl's `extract_message_text`: the body plus the text of rich
|
||||||
|
embeds and polls, newline-joined, empties dropped."""
|
||||||
|
parts = [message.get("content") or ""]
|
||||||
|
for embed in message.get("embeds") or []:
|
||||||
|
if embed.get("type") != "rich":
|
||||||
|
continue
|
||||||
|
parts.append((embed.get("author") or {}).get("name") or "")
|
||||||
|
parts.append(embed.get("title") or "")
|
||||||
|
parts.append(embed.get("description") or "")
|
||||||
|
for fld in embed.get("fields") or []:
|
||||||
|
parts.append(fld.get("name") or "")
|
||||||
|
parts.append(fld.get("value") or "")
|
||||||
|
parts.append((embed.get("footer") or {}).get("text") or "")
|
||||||
|
poll = message.get("poll")
|
||||||
|
if poll:
|
||||||
|
parts.append(((poll.get("question") or {}).get("text")) or "")
|
||||||
|
for answer in poll.get("answers") or []:
|
||||||
|
parts.append(((answer.get("poll_media") or {}).get("text")) or "")
|
||||||
|
return "\n".join(p for p in parts if p)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MediaItem:
|
||||||
|
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||||
|
split of the URL; `num` is its 1-based position across the message's files,
|
||||||
|
which is what names it on disk.
|
||||||
|
|
||||||
|
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||||||
|
`num`: an edit that removes a file renumbers the ones after it, and a
|
||||||
|
positional key would then call a different file seen. It is the
|
||||||
|
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||||||
|
the query string is a signature that changes on every fetch. `filehash` is
|
||||||
|
always None; nothing in a signed CDN URL is a content hash."""
|
||||||
|
|
||||||
|
url: str
|
||||||
|
filename: str
|
||||||
|
extension: str
|
||||||
|
kind: str
|
||||||
|
post_id: str
|
||||||
|
num: int
|
||||||
|
media_id: str
|
||||||
|
filehash: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordClient:
|
||||||
|
"""Synchronous Discord API v10 read client for one user token."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
token: str | None,
|
||||||
|
*,
|
||||||
|
request_sleep: float = 0.0,
|
||||||
|
max_retries: int = _MAX_429_RETRIES,
|
||||||
|
session: requests.Session | None = None,
|
||||||
|
):
|
||||||
|
self._session = session or requests.Session()
|
||||||
|
self._session.headers.update({
|
||||||
|
"User-Agent": firefox_user_agent(),
|
||||||
|
"Accept": "*/*",
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Referer": _ROOT + "/",
|
||||||
|
})
|
||||||
|
if token:
|
||||||
|
self._session.headers["Authorization"] = token
|
||||||
|
self._token = token
|
||||||
|
self._request_sleep = request_sleep or 0.0
|
||||||
|
self._max_retries = max_retries
|
||||||
|
self._server: dict = {}
|
||||||
|
self._channels: dict[str, dict] = {}
|
||||||
|
self._skip_feed = False
|
||||||
|
|
||||||
|
# -- request -----------------------------------------------------------
|
||||||
|
|
||||||
|
def _get(self, endpoint: str, params: dict | None = None):
|
||||||
|
if not self._token:
|
||||||
|
raise DiscordAuthError("No Discord token is configured for this source")
|
||||||
|
if self._request_sleep > 0:
|
||||||
|
time.sleep(self._request_sleep)
|
||||||
|
url = API_ROOT + endpoint
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
|
||||||
|
if resp.status_code == 429 and attempt < self._max_retries:
|
||||||
|
attempt += 1
|
||||||
|
delay = retry_after_seconds(
|
||||||
|
resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
|
||||||
|
)
|
||||||
|
log.warning(
|
||||||
|
"Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
|
||||||
|
endpoint, delay, attempt, self._max_retries,
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
continue
|
||||||
|
break
|
||||||
|
if resp.status_code == 401:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
"Discord rejected the token (HTTP 401) — it is invalid or has "
|
||||||
|
"expired; copy a fresh one from the browser",
|
||||||
|
status_code=401,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Discord returned HTTP {resp.status_code} ({endpoint})",
|
||||||
|
status_code=resp.status_code,
|
||||||
|
retry_after=_retry_after(resp),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return resp.json()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise DiscordDriftError(
|
||||||
|
f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
# -- metadata (gallery-dl parse_server / parse_channel) -----------------
|
||||||
|
|
||||||
|
def _load_server(self, server_id: str) -> None:
|
||||||
|
server = self._get(f"/guilds/{server_id}")
|
||||||
|
if not isinstance(server, dict) or "id" not in server:
|
||||||
|
raise DiscordDriftError(f"Discord server {server_id} came back without an id")
|
||||||
|
self._server = {
|
||||||
|
"server": server.get("name") or "",
|
||||||
|
"server_id": str(server["id"]),
|
||||||
|
"owner_id": server.get("owner_id"),
|
||||||
|
}
|
||||||
|
channels = self._get(f"/guilds/{server_id}/channels")
|
||||||
|
if not isinstance(channels, list):
|
||||||
|
raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
|
||||||
|
# Categories first, so every child can name its parent.
|
||||||
|
for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
|
||||||
|
self._parse_channel(channel)
|
||||||
|
|
||||||
|
def _parse_channel(self, channel: dict) -> dict:
|
||||||
|
parent_id = channel.get("parent_id")
|
||||||
|
meta = {
|
||||||
|
"channel": channel.get("name") or "",
|
||||||
|
"channel_id": str(channel.get("id")),
|
||||||
|
"channel_type": channel.get("type"),
|
||||||
|
"channel_topic": channel.get("topic") or "",
|
||||||
|
"parent_id": parent_id,
|
||||||
|
"is_thread": "thread_metadata" in channel,
|
||||||
|
}
|
||||||
|
parent = self._channels.get(parent_id) if parent_id else None
|
||||||
|
if parent:
|
||||||
|
meta["parent"] = parent["channel"]
|
||||||
|
meta["parent_type"] = parent["channel_type"]
|
||||||
|
if meta["channel_type"] in {1, 3}:
|
||||||
|
recipients = channel.get("recipients") or []
|
||||||
|
meta["channel"] = "DMs"
|
||||||
|
meta["recipients"] = [u.get("username") for u in recipients]
|
||||||
|
meta["recipients_id"] = [u.get("id") for u in recipients]
|
||||||
|
self._channels[meta["channel_id"]] = meta
|
||||||
|
return meta
|
||||||
|
|
||||||
|
def _channel_meta(self, channel_id: str) -> dict:
|
||||||
|
if channel_id not in self._channels:
|
||||||
|
self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||||
|
return self._channels[channel_id]
|
||||||
|
|
||||||
|
def _threads(self, channel_id: str) -> list[dict]:
|
||||||
|
"""Every thread of a channel or forum, newest-created first (see the
|
||||||
|
module docstring for why not last-message order)."""
|
||||||
|
threads: list[dict] = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
data = self._get(f"/channels/{channel_id}/threads/search", {
|
||||||
|
"sort_by": "last_message_time",
|
||||||
|
"sort_order": "desc",
|
||||||
|
"limit": _THREADS_BATCH,
|
||||||
|
"offset": offset,
|
||||||
|
})
|
||||||
|
batch = (data.get("threads") or []) if isinstance(data, dict) else []
|
||||||
|
threads.extend(batch)
|
||||||
|
if len(batch) < _THREADS_BATCH:
|
||||||
|
break
|
||||||
|
offset += len(batch)
|
||||||
|
threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
|
||||||
|
return threads
|
||||||
|
|
||||||
|
# -- the walk ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
|
||||||
|
"""`(channel_id, safe)` for every message feed under `channel_id`, in
|
||||||
|
gallery-dl's order. `safe` feeds are skipped on a 403."""
|
||||||
|
try:
|
||||||
|
ctype = self._channel_meta(channel_id)["channel_type"]
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
if not safe:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
f"The Discord token cannot see channel {channel_id} (HTTP 403)",
|
||||||
|
status_code=403,
|
||||||
|
) from exc
|
||||||
|
log.info("Discord: no access to channel %s — skipped", channel_id)
|
||||||
|
return
|
||||||
|
if ctype in _TEXT or ctype in _DIRECT:
|
||||||
|
yield channel_id, safe
|
||||||
|
if ctype in _TEXT or ctype in _FORUM:
|
||||||
|
try:
|
||||||
|
threads = self._threads(channel_id)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
log.info("Discord: cannot list threads of %s — skipped", channel_id)
|
||||||
|
threads = []
|
||||||
|
for thread in threads:
|
||||||
|
yield self._parse_channel(thread)["channel_id"], True
|
||||||
|
elif ctype == _CATEGORY:
|
||||||
|
for child in list(self._channels.values()):
|
||||||
|
if child.get("parent_id") == channel_id:
|
||||||
|
yield from self._feeds(child["channel_id"], safe=True)
|
||||||
|
elif ctype not in _DIRECT and not safe:
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Discord channel {channel_id} is of type {ctype}, which has no messages"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
|
||||||
|
server_id, channel_id = parse_source_url(url)
|
||||||
|
self._server, self._channels = {}, {}
|
||||||
|
if server_id is not None:
|
||||||
|
self._load_server(server_id)
|
||||||
|
if channel_id is not None:
|
||||||
|
yield from self._feeds(channel_id, safe=False)
|
||||||
|
return
|
||||||
|
for meta in list(self._channels.values()):
|
||||||
|
if meta["channel_type"] in _SERVER_WALK:
|
||||||
|
yield from self._feeds(meta["channel_id"], safe=True)
|
||||||
|
|
||||||
|
def skip_feed(self) -> None:
|
||||||
|
"""Optional core seam (#4413): end the current channel and go on to the
|
||||||
|
next one. A tick's early-out means THIS channel has nothing new, not
|
||||||
|
that the server has nothing new."""
|
||||||
|
self._skip_feed = True
|
||||||
|
|
||||||
|
def iter_posts(
|
||||||
|
self, campaign_id: str, cursor: str | None = None
|
||||||
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||||
|
"""Yield `(message, channel_meta, page_cursor)` for every content
|
||||||
|
message the source reaches, channel by channel, each newest first.
|
||||||
|
|
||||||
|
`campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
|
||||||
|
— the channel and the `before` id that fetched the page (empty for a
|
||||||
|
channel's first page) — so a backfill resumes inside the right channel
|
||||||
|
and re-fetches the page it was cut in. A cursor naming a channel the
|
||||||
|
walk no longer reaches (a deleted thread) restarts from the top rather
|
||||||
|
than walking nothing.
|
||||||
|
"""
|
||||||
|
resume_channel, _, resume_before = (cursor or "").partition(":")
|
||||||
|
resuming = bool(resume_channel)
|
||||||
|
feeds = list(self._source_feeds(campaign_id)) if resuming else None
|
||||||
|
if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
|
||||||
|
log.warning(
|
||||||
|
"Discord: resume channel %s is no longer in %s — restarting",
|
||||||
|
resume_channel, campaign_id,
|
||||||
|
)
|
||||||
|
resuming = False
|
||||||
|
for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
|
||||||
|
before = None
|
||||||
|
if resuming:
|
||||||
|
if channel_id != resume_channel:
|
||||||
|
continue
|
||||||
|
resuming = False
|
||||||
|
before = resume_before or None
|
||||||
|
yield from self._iter_channel(channel_id, before, safe=safe)
|
||||||
|
|
||||||
|
def _iter_channel(
|
||||||
|
self, channel_id: str, before: str | None, *, safe: bool
|
||||||
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||||
|
self._skip_feed = False
|
||||||
|
meta = {**self._server, **self._channels.get(channel_id, {})}
|
||||||
|
while True:
|
||||||
|
page_cursor = f"{channel_id}:{before or ''}"
|
||||||
|
try:
|
||||||
|
messages = self._get(
|
||||||
|
f"/channels/{channel_id}/messages",
|
||||||
|
{"limit": _MESSAGES_BATCH, "before": before},
|
||||||
|
)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
if not safe:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
f"The Discord token cannot read channel {channel_id} (HTTP 403)",
|
||||||
|
status_code=403,
|
||||||
|
) from exc
|
||||||
|
log.info("Discord: no access to messages of %s — skipped", channel_id)
|
||||||
|
return
|
||||||
|
if not isinstance(messages, list):
|
||||||
|
raise DiscordDriftError(
|
||||||
|
f"Discord messages of {channel_id} came back as "
|
||||||
|
f"{type(messages).__name__}, not a list"
|
||||||
|
)
|
||||||
|
for message in messages:
|
||||||
|
if message.get("type") not in MESSAGE_TYPES:
|
||||||
|
continue
|
||||||
|
message["_meta"] = meta
|
||||||
|
yield message, meta, page_cursor
|
||||||
|
if self._skip_feed:
|
||||||
|
return
|
||||||
|
if len(messages) < _MESSAGES_BATCH:
|
||||||
|
return
|
||||||
|
before = str(messages[-1]["id"])
|
||||||
|
|
||||||
|
# -- per-message -------------------------------------------------------
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||||
|
"""gallery-dl's file list for one message: attachments, then the first
|
||||||
|
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||||
|
the same for every forwarded snapshot; numbered from 1 across them."""
|
||||||
|
mid = str(post.get("id") or "")
|
||||||
|
snapshots = [post] + [
|
||||||
|
(s or {}).get("message") or {}
|
||||||
|
for s in post.get("message_snapshots") or []
|
||||||
|
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||||
|
]
|
||||||
|
found: list[tuple[str, str, str | None]] = []
|
||||||
|
for snap in snapshots:
|
||||||
|
for att in snap.get("attachments") or []:
|
||||||
|
if att.get("url"):
|
||||||
|
aid = att.get("id")
|
||||||
|
found.append((att["url"], "attachment", str(aid) if aid else None))
|
||||||
|
for embed in snap.get("embeds") or []:
|
||||||
|
if embed.get("type") not in _EMBED_TYPES:
|
||||||
|
continue
|
||||||
|
for fld in ("video", "image", "thumbnail"):
|
||||||
|
url = (embed.get(fld) or {}).get("proxy_url")
|
||||||
|
if url:
|
||||||
|
found.append((url, "embed", None))
|
||||||
|
break
|
||||||
|
items = []
|
||||||
|
for num, (url, kind, fid) in enumerate(found, start=1):
|
||||||
|
name, ext = nameext_from_url(url)
|
||||||
|
if fid is None:
|
||||||
|
path = url.partition("?")[0].encode()
|
||||||
|
fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
|
||||||
|
items.append(MediaItem(
|
||||||
|
url=url, filename=name, extension=ext, kind=kind, post_id=mid,
|
||||||
|
num=num, media_id=fid,
|
||||||
|
))
|
||||||
|
return items
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def post_meta(post: dict) -> dict:
|
||||||
|
"""No title (Discord has none); `date` is the message timestamp, ISO
|
||||||
|
with an offset — what the core's revisit window reads."""
|
||||||
|
return {"title": None, "date": post.get("timestamp")}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def post_record_key(cls, post: dict) -> tuple[str, str] | None:
|
||||||
|
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||||
|
ledger, like `post:<id>` on the other platforms.
|
||||||
|
|
||||||
|
None for a message with no files. gallery-dl wrote a sidecar only
|
||||||
|
beside a file, so a text-only chat line never became a post, and the
|
||||||
|
drop grouping (discord_grouping) is built on that: a channel's chatter
|
||||||
|
recorded as posts would bury the drops it exists to surface."""
|
||||||
|
mid = post.get("id")
|
||||||
|
mid = str(mid) if mid is not None else ""
|
||||||
|
if not mid or not cls.extract_media(post):
|
||||||
|
return None
|
||||||
|
return (f"message:{mid}", mid)
|
||||||
|
|
||||||
|
# -- verify ------------------------------------------------------------
|
||||||
|
|
||||||
|
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||||
|
"""Is the token valid, and can it see what the source names?"""
|
||||||
|
try:
|
||||||
|
server_id, channel_id = parse_source_url(url)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
return None, str(exc)
|
||||||
|
try:
|
||||||
|
me = self._get("/users/@me")
|
||||||
|
if channel_id is not None:
|
||||||
|
self._get(f"/channels/{channel_id}")
|
||||||
|
elif server_id is not None:
|
||||||
|
self._get(f"/guilds/{server_id}")
|
||||||
|
except DiscordAuthError as exc:
|
||||||
|
return False, f"Discord rejected the token — {exc}"
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code in (403, 404):
|
||||||
|
return False, (
|
||||||
|
"The token is valid, but its account cannot see "
|
||||||
|
f"{'this channel' if channel_id else 'this server'} "
|
||||||
|
f"(HTTP {exc.status_code})"
|
||||||
|
)
|
||||||
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||||
|
who = (me or {}).get("username") if isinstance(me, dict) else None
|
||||||
|
return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_after(resp: requests.Response) -> float | None:
|
||||||
|
hdr = resp.headers.get("Retry-After")
|
||||||
|
try:
|
||||||
|
return float(hdr) if hdr else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""Native Discord media downloader — the Discord counterpart to
|
||||||
|
subscribestar_downloader.
|
||||||
|
|
||||||
|
Writes files exactly where gallery-dl wrote them, so a cutover finds every
|
||||||
|
existing file on disk (`skipped_disk`) instead of fetching it again:
|
||||||
|
|
||||||
|
<images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
|
||||||
|
|
||||||
|
That is FC's gallery-dl config (`gallery_dl.DISCORD_DIRECTORY` /
|
||||||
|
`DISCORD_FILENAME`) under the per-source base directory
|
||||||
|
`<images>/<artist>/<platform>`. The name is cleaned the way gallery-dl cleans it
|
||||||
|
on Linux — `/` becomes `_` and control characters are removed, nothing else
|
||||||
|
(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
|
||||||
|
whose Windows set would turn a `:` in a channel or file name into `_` and miss
|
||||||
|
the file gallery-dl wrote.
|
||||||
|
|
||||||
|
Post-first (rule 120): each file gets a minimal sidecar named like it minus the
|
||||||
|
extension (what `find_sidecar` pairs first), and the message itself gets one
|
||||||
|
record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
|
||||||
|
— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
|
||||||
|
`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
|
||||||
|
the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
|
||||||
|
outrank `message_id` in the post-id chain (`platforms.base`).
|
||||||
|
|
||||||
|
PURE: no DB; the seen-skip is an injected predicate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .discord_client import firefox_user_agent, message_text
|
||||||
|
from .native_ingest_common import (
|
||||||
|
BaseNativeDownloader,
|
||||||
|
MediaOutcome,
|
||||||
|
PostRecordOutcome,
|
||||||
|
make_session,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
PLATFORM = "discord"
|
||||||
|
_CONTROL = re.compile("[\x00-\x1f\x7f]")
|
||||||
|
# gallery-dl falls back to the response's type for a URL with no extension;
|
||||||
|
# we never see the response before naming, and such URLs do not occur for
|
||||||
|
# Discord attachments or embed proxies in practice.
|
||||||
|
_NO_EXTENSION = "bin"
|
||||||
|
|
||||||
|
|
||||||
|
def gdl_clean(segment: str) -> str:
|
||||||
|
"""One path segment as gallery-dl writes it on Linux."""
|
||||||
|
return _CONTROL.sub("", segment.replace("/", "_"))
|
||||||
|
|
||||||
|
|
||||||
|
def message_date(post: dict) -> datetime | None:
|
||||||
|
raw = post.get("timestamp")
|
||||||
|
if not isinstance(raw, str) or not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
|
||||||
|
"""gallery-dl's `{channel}` directory; an empty name adds no segment."""
|
||||||
|
base = Path(images_root) / artist_slug / PLATFORM
|
||||||
|
channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
|
||||||
|
return base / channel if channel else base
|
||||||
|
|
||||||
|
|
||||||
|
def media_stem(post: dict, media) -> str:
|
||||||
|
"""`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
|
||||||
|
when = message_date(post)
|
||||||
|
day = f"{when:%Y%m%d}" if when else "None"
|
||||||
|
return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordDownloader(BaseNativeDownloader):
|
||||||
|
"""Download a message's files to gallery-dl's layout. The CDN gets
|
||||||
|
gallery-dl's browser profile and no token — gallery-dl sends the token only
|
||||||
|
to the API, and the CDN URLs are pre-signed."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
images_root: Path,
|
||||||
|
cookies_path: str | None = None,
|
||||||
|
*,
|
||||||
|
validate: bool = True,
|
||||||
|
rate_limit: float = 0.0,
|
||||||
|
session: requests.Session | None = None,
|
||||||
|
):
|
||||||
|
super().__init__(
|
||||||
|
images_root, None, platform=PLATFORM,
|
||||||
|
validate=validate, rate_limit=rate_limit,
|
||||||
|
session=session if session is not None else make_session(None, extra_headers={
|
||||||
|
"User-Agent": firefox_user_agent(),
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Referer": "https://discord.com/",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
def download_post(
|
||||||
|
self,
|
||||||
|
post: dict,
|
||||||
|
media_items: list,
|
||||||
|
artist_slug: str,
|
||||||
|
*,
|
||||||
|
is_seen: Callable[[object], bool] = lambda m: False,
|
||||||
|
should_stop: Callable[[], bool] = lambda: False,
|
||||||
|
recapture: bool = False,
|
||||||
|
) -> list[MediaOutcome]:
|
||||||
|
"""Every file of one message; per-file outcomes, one failure isolated."""
|
||||||
|
folder = channel_dir(self.images_root, artist_slug, post)
|
||||||
|
outcomes: list[MediaOutcome] = []
|
||||||
|
for media in media_items:
|
||||||
|
if should_stop():
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
outcomes.append(self._download_one(
|
||||||
|
post, media, folder, artist_slug, is_seen, recapture=recapture,
|
||||||
|
))
|
||||||
|
except Exception as exc: # resilient: isolate one item's failure
|
||||||
|
log.warning(
|
||||||
|
"Discord media failed (message %s, file %d): %s",
|
||||||
|
post.get("id"), media.num, exc,
|
||||||
|
)
|
||||||
|
outcomes.append(
|
||||||
|
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||||
|
)
|
||||||
|
return outcomes
|
||||||
|
|
||||||
|
def _download_one(
|
||||||
|
self,
|
||||||
|
post: dict,
|
||||||
|
media,
|
||||||
|
folder: Path,
|
||||||
|
artist_slug: str,
|
||||||
|
is_seen: Callable[[object], bool],
|
||||||
|
*,
|
||||||
|
recapture: bool = False,
|
||||||
|
) -> MediaOutcome:
|
||||||
|
seen = is_seen(media)
|
||||||
|
if seen and not recapture:
|
||||||
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
stem = media_stem(post, media)
|
||||||
|
path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
|
||||||
|
if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it
|
||||||
|
return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
|
||||||
|
if seen: # recapture never re-fetches a seen file that is gone
|
||||||
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
|
||||||
|
folder.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self._rate_limit > 0:
|
||||||
|
time.sleep(self._rate_limit)
|
||||||
|
out = self._fetch_get(media.url, path)
|
||||||
|
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||||
|
if reason is not None:
|
||||||
|
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||||
|
sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
|
||||||
|
sidecar["source_url"] = media.url
|
||||||
|
(folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
|
||||||
|
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||||
|
|
||||||
|
def write_post_record(
|
||||||
|
self, post: dict, artist_slug: str, *, revisit: bool = False,
|
||||||
|
) -> PostRecordOutcome:
|
||||||
|
"""The message record — the one writer of a Discord post's body, date
|
||||||
|
and permalink ids. `revisit` re-reads a message already captured (an
|
||||||
|
edit); an empty re-read writes nothing, so it never blanks a body."""
|
||||||
|
mid = str(post.get("id") or "")
|
||||||
|
body = message_text(post)
|
||||||
|
if not mid or (revisit and not body.strip()):
|
||||||
|
return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
|
||||||
|
meta = post.get("_meta") or {}
|
||||||
|
author = post.get("author") or {}
|
||||||
|
record = {
|
||||||
|
"category": PLATFORM,
|
||||||
|
"message_id": mid,
|
||||||
|
"server": meta.get("server"),
|
||||||
|
"server_id": meta.get("server_id"),
|
||||||
|
"channel": meta.get("channel"),
|
||||||
|
"channel_id": meta.get("channel_id") or post.get("channel_id"),
|
||||||
|
"parent": meta.get("parent"),
|
||||||
|
"is_thread": meta.get("is_thread"),
|
||||||
|
"author": author.get("username"),
|
||||||
|
"author_id": author.get("id"),
|
||||||
|
"message": body,
|
||||||
|
"date": post.get("timestamp"),
|
||||||
|
}
|
||||||
|
folder = channel_dir(self.images_root, artist_slug, post)
|
||||||
|
folder.mkdir(parents=True, exist_ok=True)
|
||||||
|
when = message_date(post)
|
||||||
|
day = f"{when:%Y%m%d}" if when else "None"
|
||||||
|
path = folder / f"{day}_{mid}_post.json"
|
||||||
|
path.write_text(json.dumps(
|
||||||
|
{k: v for k, v in record.items() if v is not None}, indent=2,
|
||||||
|
))
|
||||||
|
return PostRecordOutcome(
|
||||||
|
path=path, post_type=None, title=None, body_chars=len(body),
|
||||||
|
)
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`.
|
||||||
|
|
||||||
|
Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes,
|
||||||
|
both ledgers, cursor checkpointing and the post-first capture live in the core;
|
||||||
|
this wires in the Discord client, downloader, ledger models and key.
|
||||||
|
|
||||||
|
Two things differ from the cookie platforms:
|
||||||
|
|
||||||
|
- Discord authenticates with a user TOKEN, so `auth_token` is the credential
|
||||||
|
here rather than an argument accepted and ignored.
|
||||||
|
- The body canary is off. It fails a walk whose first 30+ captured posts all
|
||||||
|
came back without text, on the theory that a creator nearly always writes
|
||||||
|
something; a Discord drop is routinely files and nothing else, so on
|
||||||
|
Discord that is an ordinary backfill, not a broken parser.
|
||||||
|
|
||||||
|
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||||
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..models import DiscordFailedMedia, DiscordSeenMedia
|
||||||
|
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||||
|
from .discord_downloader import DiscordDownloader
|
||||||
|
from .ingest_core import Ingester
|
||||||
|
|
||||||
|
_LEDGER_KEY_MAX = 128
|
||||||
|
|
||||||
|
|
||||||
|
def _ledger_key(media: MediaItem) -> str:
|
||||||
|
"""`<message_id>:<media_id>` — stable across edits (see MediaItem)."""
|
||||||
|
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordIngester(Ingester):
|
||||||
|
"""Walk a Discord source's channels, download unseen files, return a
|
||||||
|
`DownloadResult`. `client` / `downloader` are injectable for tests."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
images_root: Path,
|
||||||
|
cookies_path: str | None,
|
||||||
|
session_factory: Callable[[], object],
|
||||||
|
*,
|
||||||
|
validate: bool = True,
|
||||||
|
rate_limit: float = 0.0,
|
||||||
|
request_sleep: float = 0.0,
|
||||||
|
auth_token: str | None = None,
|
||||||
|
client: DiscordClient | None = None,
|
||||||
|
downloader: DiscordDownloader | None = None,
|
||||||
|
):
|
||||||
|
del cookies_path # Discord authenticates by token (uniform signature)
|
||||||
|
self.images_root = Path(images_root)
|
||||||
|
super().__init__(
|
||||||
|
client=client if client is not None else DiscordClient(
|
||||||
|
auth_token, request_sleep=request_sleep,
|
||||||
|
),
|
||||||
|
downloader=downloader if downloader is not None else DiscordDownloader(
|
||||||
|
self.images_root, validate=validate, rate_limit=rate_limit,
|
||||||
|
),
|
||||||
|
session_factory=session_factory,
|
||||||
|
seen_model=DiscordSeenMedia,
|
||||||
|
failed_model=DiscordFailedMedia,
|
||||||
|
seen_constraint="uq_discord_seen_media_source_id",
|
||||||
|
failed_constraint="uq_discord_failed_media_source_id",
|
||||||
|
ledger_key=_ledger_key,
|
||||||
|
platform="discord",
|
||||||
|
error_base=DiscordAPIError,
|
||||||
|
drift_label="Discord API",
|
||||||
|
body_canary=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||||
|
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||||
|
account see the channel or server the source names?"""
|
||||||
|
if not auth_token:
|
||||||
|
return False, "No Discord token is saved — add one under Credentials."
|
||||||
|
client = DiscordClient(auth_token)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
return await loop.run_in_executor(None, client.verify_auth, url)
|
||||||
@@ -23,6 +23,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .discord_ingester import DiscordIngester
|
||||||
from .gallery_dl import DownloadResult, ErrorType
|
from .gallery_dl import DownloadResult, ErrorType
|
||||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||||
from .patreon_ingester import PatreonIngester
|
from .patreon_ingester import PatreonIngester
|
||||||
@@ -31,9 +32,13 @@ from .platforms import known_platform_keys
|
|||||||
from .subscribestar_ingester import SubscribeStarIngester
|
from .subscribestar_ingester import SubscribeStarIngester
|
||||||
|
|
||||||
# Platforms whose download + verify go through the native ingester rather than
|
# Platforms whose download + verify go through the native ingester rather than
|
||||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
|
# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it
|
||||||
# they migrate too.
|
# migrates too. Discord joined in milestone 428.
|
||||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||||
|
|
||||||
|
# Native platforms whose feed id IS the source URL, so there is nothing to
|
||||||
|
# resolve: SubscribeStar's creator page, Discord's server/channel link.
|
||||||
|
_URL_IS_FEED = frozenset({"subscribestar", "discord"})
|
||||||
|
|
||||||
|
|
||||||
def _unsupported_platform_message(platform: str) -> str | None:
|
def _unsupported_platform_message(platform: str) -> str | None:
|
||||||
@@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str):
|
|||||||
return {
|
return {
|
||||||
"patreon": PatreonIngester,
|
"patreon": PatreonIngester,
|
||||||
"subscribestar": SubscribeStarIngester,
|
"subscribestar": SubscribeStarIngester,
|
||||||
|
"discord": DiscordIngester,
|
||||||
}[platform]
|
}[platform]
|
||||||
|
|
||||||
|
|
||||||
@@ -126,10 +132,10 @@ async def _resolve_native_campaign_id(
|
|||||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None]:
|
||||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
and Discord's feed id IS the source URL (no lookup → resolved None). Patreon
|
||||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
resolves the campaign id from the vanity URL (resolved non-None when a lookup
|
||||||
so phase 3 caches it)."""
|
actually ran, so phase 3 caches it)."""
|
||||||
if platform == "subscribestar":
|
if platform in _URL_IS_FEED:
|
||||||
return url, None
|
return url, None
|
||||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||||
|
|
||||||
@@ -252,6 +258,10 @@ async def verify_source_credential(
|
|||||||
from .subscribestar_ingester import verify_subscribestar_credential
|
from .subscribestar_ingester import verify_subscribestar_credential
|
||||||
|
|
||||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||||
|
if platform == "discord":
|
||||||
|
from .discord_ingester import verify_discord_credential
|
||||||
|
|
||||||
|
return await verify_discord_credential(url, auth_token)
|
||||||
from .patreon_ingester import verify_patreon_credential
|
from .patreon_ingester import verify_patreon_credential
|
||||||
|
|
||||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||||
|
|||||||
@@ -264,6 +264,14 @@ class Ingester:
|
|||||||
# operator-driven "re-read every body" pass; a horizon there would be a
|
# operator-driven "re-read every body" pass; a horizon there would be a
|
||||||
# third overlapping answer to a question that has two.
|
# third overlapping answer to a question that has two.
|
||||||
post_meta = getattr(self.client, "post_meta", None)
|
post_meta = getattr(self.client, "post_meta", None)
|
||||||
|
# #4413: optional client seam for a source that is several feeds walked
|
||||||
|
# one after another (a Discord server: every channel and thread). The
|
||||||
|
# tick early-out means "this feed has nothing new", and without the
|
||||||
|
# seam it ends the WHOLE walk — so the first quiet channel would hide
|
||||||
|
# every channel after it. With it, the early-out asks the client to
|
||||||
|
# move on and the walk continues. Absent → the early-out ends the walk,
|
||||||
|
# exactly as before (Patreon and SubscribeStar are one feed each).
|
||||||
|
skip_feed = getattr(self.client, "skip_feed", None)
|
||||||
horizon: datetime | None = None
|
horizon: datetime | None = None
|
||||||
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
||||||
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
||||||
@@ -312,6 +320,7 @@ class Ingester:
|
|||||||
reached_bottom = False
|
reached_bottom = False
|
||||||
budget_hit = False
|
budget_hit = False
|
||||||
early_out = False
|
early_out = False
|
||||||
|
feeds_caught_up = 0 # #4413: feeds a tick left early via skip_feed
|
||||||
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
||||||
cancel_armed = False # latched once we observe a live "running" state
|
cancel_armed = False # latched once we observe a live "running" state
|
||||||
|
|
||||||
@@ -674,9 +683,15 @@ class Ingester:
|
|||||||
})
|
})
|
||||||
|
|
||||||
if early_out:
|
if early_out:
|
||||||
break
|
if skip_feed is None:
|
||||||
|
break
|
||||||
|
skip_feed()
|
||||||
|
feeds_caught_up += 1
|
||||||
|
early_out = False
|
||||||
|
consecutive_seen = 0
|
||||||
else:
|
else:
|
||||||
reached_bottom = True
|
# A walk that left feeds early did not read to their ends.
|
||||||
|
reached_bottom = not feeds_caught_up
|
||||||
except self._error_base as exc:
|
except self._error_base as exc:
|
||||||
# The platform's client-error base — _failure_result (adapter)
|
# The platform's client-error base — _failure_result (adapter)
|
||||||
# maps it to a typed error.
|
# maps it to a typed error.
|
||||||
@@ -724,6 +739,7 @@ class Ingester:
|
|||||||
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
||||||
if revisited else ""
|
if revisited else ""
|
||||||
)
|
)
|
||||||
|
+ (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "")
|
||||||
+ (", reached end" if reached_bottom else "")
|
+ (", reached end" if reached_bottom else "")
|
||||||
+ (", time-boxed" if budget_hit else "")
|
+ (", time-boxed" if budget_hit else "")
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ log = logging.getLogger(__name__)
|
|||||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||||
# Add a platform here to cap it to a single concurrent walk.
|
# Add a platform here to cap it to a single concurrent walk. Discord most of
|
||||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
# all: every source walks on the operator's ONE user token, and parallel walks
|
||||||
|
# on a user account are both how its rate limit trips and what gets it flagged.
|
||||||
|
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||||
|
|
||||||
_LOCK_PREFIX = "fc:download_lock:"
|
_LOCK_PREFIX = "fc:download_lock:"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,339 @@
|
|||||||
|
"""The native Discord client walks what gallery-dl walked, in its order (#4412).
|
||||||
|
|
||||||
|
No network: a fake session answers by endpoint. What these pin is the part of
|
||||||
|
gallery-dl's behaviour that decides WHICH files exist and WHAT they are called —
|
||||||
|
the channel walk, the file list and its numbering, the name split — because a
|
||||||
|
difference there is a re-download or a missed file at cutover, not a style
|
||||||
|
choice.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.app.services import discord_client as dc
|
||||||
|
from backend.app.services.discord_client import (
|
||||||
|
DiscordAPIError,
|
||||||
|
DiscordAuthError,
|
||||||
|
DiscordClient,
|
||||||
|
firefox_user_agent,
|
||||||
|
message_text,
|
||||||
|
nameext_from_url,
|
||||||
|
parse_source_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, status, body=None, headers=None):
|
||||||
|
self.status_code = status
|
||||||
|
self._body = body
|
||||||
|
self.headers = headers or {}
|
||||||
|
self.content = b"x"
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return self._body
|
||||||
|
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
"""Answers `GET API_ROOT + endpoint` from a dict. A value may be a list of
|
||||||
|
responses, served in order; a messages entry is keyed by (endpoint, before)."""
|
||||||
|
|
||||||
|
def __init__(self, routes):
|
||||||
|
self.routes = routes
|
||||||
|
self.headers = {}
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def get(self, url, params=None, timeout=None):
|
||||||
|
endpoint = url[len(dc.API_ROOT):]
|
||||||
|
self.calls.append((endpoint, dict(params or {})))
|
||||||
|
key = endpoint
|
||||||
|
if endpoint.endswith("/messages"):
|
||||||
|
key = (endpoint, (params or {}).get("before"))
|
||||||
|
elif endpoint.endswith("/threads/search"):
|
||||||
|
key = (endpoint, (params or {}).get("offset"))
|
||||||
|
answer = self.routes.get(key, _Resp(404, {}))
|
||||||
|
if isinstance(answer, list):
|
||||||
|
return answer.pop(0)
|
||||||
|
return answer
|
||||||
|
|
||||||
|
|
||||||
|
def _ok(body):
|
||||||
|
return _Resp(200, body)
|
||||||
|
|
||||||
|
|
||||||
|
def _msg(mid, *, content="", attachments=(), embeds=(), **extra):
|
||||||
|
return {
|
||||||
|
"id": str(mid), "type": 0, "content": content,
|
||||||
|
"timestamp": "2026-09-20T12:00:00.000000+00:00",
|
||||||
|
"author": {"id": "7", "username": "artist"},
|
||||||
|
"attachments": list(attachments), "embeds": list(embeds), **extra,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _client(routes, **kw):
|
||||||
|
return DiscordClient("tok", session=_Session(routes), **kw)
|
||||||
|
|
||||||
|
|
||||||
|
def _ids(client, url, cursor=None):
|
||||||
|
return [(m["id"], cur) for m, _meta, cur in client.iter_posts(url, cursor)]
|
||||||
|
|
||||||
|
|
||||||
|
# -- pure helpers --------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_source_urls():
|
||||||
|
assert parse_source_url("https://discord.com/channels/1/2") == ("1", "2")
|
||||||
|
assert parse_source_url("https://discord.com/channels/1") == ("1", None)
|
||||||
|
assert parse_source_url("https://discord.com/channels/1/2/threads/3") == ("1", "3")
|
||||||
|
assert parse_source_url("https://discord.com/channels/@me/5") == (None, "5")
|
||||||
|
assert parse_source_url("discord.com/channels/1/2/") == ("1", "2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_message_link_is_not_a_source():
|
||||||
|
with pytest.raises(DiscordAPIError):
|
||||||
|
parse_source_url("https://discord.com/channels/1/2/3")
|
||||||
|
with pytest.raises(DiscordAPIError):
|
||||||
|
parse_source_url("https://example.com/channels/1/2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_name_split_matches_gallery_dl():
|
||||||
|
url = "https://cdn.discordapp.com/attachments/1/2/My%20Pic.final.PNG?ex=a&hm=b"
|
||||||
|
assert nameext_from_url(url) == ("My Pic.final", "png")
|
||||||
|
assert nameext_from_url("https://x/y/noext") == ("noext", "")
|
||||||
|
assert nameext_from_url("https://x/y/a." + "b" * 17) == ("a." + "b" * 17, "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_agent_is_gallery_dls_dated_firefox():
|
||||||
|
"""gallery-dl's own comment: "147 on 2026-01-13"."""
|
||||||
|
assert "Firefox/147.0" in firefox_user_agent(date(2026, 1, 13))
|
||||||
|
|
||||||
|
|
||||||
|
def test_message_text_takes_rich_embeds_and_polls():
|
||||||
|
m = _msg(1, content="hello", embeds=[
|
||||||
|
{"type": "rich", "author": {"name": "A"}, "title": "T",
|
||||||
|
"fields": [{"name": "f", "value": "v"}], "footer": {"text": "ft"}},
|
||||||
|
{"type": "image", "title": "not text"},
|
||||||
|
], poll={"question": {"text": "Q?"}, "answers": [{"poll_media": {"text": "yes"}}]})
|
||||||
|
assert message_text(m) == "hello\nA\nT\nf\nv\nft\nQ?\nyes"
|
||||||
|
|
||||||
|
|
||||||
|
def test_files_are_attachments_then_embeds_then_snapshots_numbered_across():
|
||||||
|
m = _msg(9, attachments=[{"url": "https://cdn/a/1.png"}], embeds=[
|
||||||
|
{"type": "video", "video": {"proxy_url": "https://media/v.mp4"},
|
||||||
|
"thumbnail": {"proxy_url": "https://media/t.jpg"}},
|
||||||
|
{"type": "image", "thumbnail": {"proxy_url": "https://media/i.webp"}},
|
||||||
|
{"type": "rich", "image": {"proxy_url": "https://media/rich.png"}},
|
||||||
|
], message_snapshots=[
|
||||||
|
{"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.gif"}],
|
||||||
|
"embeds": []}},
|
||||||
|
{"message": {"type": 7, "attachments": [{"url": "https://cdn/a/join.png"}]}},
|
||||||
|
])
|
||||||
|
items = DiscordClient.extract_media(m)
|
||||||
|
assert [(i.num, i.filename, i.extension, i.kind) for i in items] == [
|
||||||
|
(1, "1", "png", "attachment"),
|
||||||
|
(2, "v", "mp4", "embed"),
|
||||||
|
(3, "i", "webp", "embed"),
|
||||||
|
(4, "fwd", "gif", "attachment"),
|
||||||
|
]
|
||||||
|
assert {i.post_id for i in items} == {"9"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_ledger_identity_survives_a_renumbering_edit():
|
||||||
|
"""Removing the first file renumbers the second; its identity must not move."""
|
||||||
|
a = {"id": "100", "url": "https://cdn/a/1.png?ex=1"}
|
||||||
|
b = {"id": "200", "url": "https://cdn/a/2.png?ex=1"}
|
||||||
|
before = DiscordClient.extract_media(_msg(9, attachments=[a, b]))
|
||||||
|
after = DiscordClient.extract_media(_msg(9, attachments=[b]))
|
||||||
|
assert (before[1].num, after[0].num) == (2, 1)
|
||||||
|
assert before[1].media_id == after[0].media_id == "200"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_embeds_identity_ignores_its_signature():
|
||||||
|
def embed(sig):
|
||||||
|
return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}}
|
||||||
|
|
||||||
|
one = DiscordClient.extract_media(_msg(9, embeds=[embed("a")]))[0].media_id
|
||||||
|
two = DiscordClient.extract_media(_msg(9, embeds=[embed("b")]))[0].media_id
|
||||||
|
assert one == two and len(one) <= 33
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_seams():
|
||||||
|
with_file = _msg(5, attachments=[{"url": "https://cdn/a/1.png"}])
|
||||||
|
assert DiscordClient.post_record_key(with_file) == ("message:5", "5")
|
||||||
|
assert DiscordClient.post_record_key({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_text_only_message_is_not_a_post():
|
||||||
|
"""gallery-dl never made one: chat lines would bury the drops."""
|
||||||
|
assert DiscordClient.post_record_key(_msg(6, content="brb")) is None
|
||||||
|
assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20")
|
||||||
|
|
||||||
|
|
||||||
|
# -- the walk --------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_a_channel_pages_newest_first_and_skips_system_messages():
|
||||||
|
page1 = [_msg(i) for i in range(300, 200, -1)]
|
||||||
|
page1[3]["type"] = 7 # a member-join line: not content
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||||
|
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "art"}]),
|
||||||
|
("/channels/2/messages", None): _ok(page1),
|
||||||
|
("/channels/2/messages", "201"): _ok([_msg(150)]),
|
||||||
|
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||||
|
}
|
||||||
|
got = _ids(_client(routes), "https://discord.com/channels/1/2")
|
||||||
|
assert len(got) == 100 # 99 of page 1 + 1 of page 2
|
||||||
|
assert "297" not in [mid for mid, _ in got]
|
||||||
|
assert got[0] == ("300", "2:")
|
||||||
|
assert got[-1] == ("150", "2:201")
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_carry_server_and_channel_metadata():
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "Studio", "owner_id": "9"}),
|
||||||
|
"/guilds/1/channels": _ok([
|
||||||
|
{"id": "4", "type": 4, "name": "Art"},
|
||||||
|
{"id": "2", "type": 0, "name": "drops", "parent_id": "4"},
|
||||||
|
]),
|
||||||
|
("/channels/2/messages", None): _ok([_msg(10)]),
|
||||||
|
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||||
|
}
|
||||||
|
[(msg, meta, _)] = list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||||||
|
assert msg["_meta"] is meta
|
||||||
|
assert meta["server"] == "Studio" and meta["server_id"] == "1"
|
||||||
|
assert meta["channel"] == "drops" and meta["channel_id"] == "2"
|
||||||
|
assert meta["parent"] == "Art"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private():
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||||
|
"/guilds/1/channels": _ok([
|
||||||
|
{"id": "2", "type": 0, "name": "text"},
|
||||||
|
{"id": "3", "type": 2, "name": "voice"},
|
||||||
|
{"id": "5", "type": 15, "name": "forum"},
|
||||||
|
{"id": "6", "type": 0, "name": "private"},
|
||||||
|
]),
|
||||||
|
("/channels/2/messages", None): _ok([_msg(20)]),
|
||||||
|
("/channels/2/threads/search", 0): _ok({"threads": [
|
||||||
|
{"id": "21", "type": 11, "name": "old", "parent_id": "2", "thread_metadata": {}},
|
||||||
|
{"id": "22", "type": 11, "name": "new", "parent_id": "2", "thread_metadata": {}},
|
||||||
|
]}),
|
||||||
|
("/channels/22/messages", None): _ok([_msg(220)]),
|
||||||
|
("/channels/21/messages", None): _Resp(403, {}), # a private thread
|
||||||
|
("/channels/5/threads/search", 0): _ok({"threads": [
|
||||||
|
{"id": "51", "type": 11, "name": "post", "parent_id": "5", "thread_metadata": {}},
|
||||||
|
]}),
|
||||||
|
("/channels/51/messages", None): _ok([_msg(510)]),
|
||||||
|
("/channels/6/messages", None): _Resp(403, {}),
|
||||||
|
("/channels/6/threads/search", 0): _Resp(403, {}),
|
||||||
|
}
|
||||||
|
got = [mid for mid, _ in _ids(_client(routes), "https://discord.com/channels/1")]
|
||||||
|
assert got == ["20", "220", "510"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_resume_cursor_reenters_its_channel_at_its_page():
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||||
|
"/guilds/1/channels": _ok([
|
||||||
|
{"id": "2", "type": 0, "name": "a"},
|
||||||
|
{"id": "3", "type": 0, "name": "b"},
|
||||||
|
]),
|
||||||
|
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||||
|
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||||||
|
("/channels/3/messages", "77"): _ok([_msg(70)]),
|
||||||
|
}
|
||||||
|
client = _client(routes)
|
||||||
|
assert _ids(client, "https://discord.com/channels/1", "3:77") == [("70", "3:77")]
|
||||||
|
fetched = [c for c in client._session.calls if c[0].endswith("/messages")]
|
||||||
|
assert fetched == [("/channels/3/messages", {"limit": 100, "before": "77"})]
|
||||||
|
|
||||||
|
|
||||||
|
def test_skip_feed_ends_the_channel_not_the_walk():
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||||
|
"/guilds/1/channels": _ok([
|
||||||
|
{"id": "2", "type": 0, "name": "a"},
|
||||||
|
{"id": "3", "type": 0, "name": "b"},
|
||||||
|
]),
|
||||||
|
("/channels/2/messages", None): _ok([_msg(29), _msg(28)]),
|
||||||
|
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||||
|
("/channels/3/messages", None): _ok([_msg(39)]),
|
||||||
|
("/channels/3/threads/search", 0): _ok({"threads": []}),
|
||||||
|
}
|
||||||
|
client = _client(routes)
|
||||||
|
seen = []
|
||||||
|
for msg, _meta, _cur in client.iter_posts("https://discord.com/channels/1"):
|
||||||
|
seen.append(msg["id"])
|
||||||
|
if msg["id"] == "29":
|
||||||
|
client.skip_feed()
|
||||||
|
assert seen == ["29", "39"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_named_channel_refusing_the_token_is_an_auth_failure():
|
||||||
|
routes = {
|
||||||
|
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||||
|
"/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "a"}]),
|
||||||
|
("/channels/2/messages", None): _Resp(403, {}),
|
||||||
|
}
|
||||||
|
with pytest.raises(DiscordAuthError):
|
||||||
|
list(_client(routes).iter_posts("https://discord.com/channels/1/2"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_401_is_an_invalid_token():
|
||||||
|
with pytest.raises(DiscordAuthError):
|
||||||
|
list(_client({"/guilds/1": _Resp(401, {})}).iter_posts(
|
||||||
|
"https://discord.com/channels/1"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_token_fails_before_any_request():
|
||||||
|
client = DiscordClient(None, session=_Session({}))
|
||||||
|
with pytest.raises(DiscordAuthError):
|
||||||
|
list(client.iter_posts("https://discord.com/channels/1"))
|
||||||
|
assert client._session.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_429_waits_and_retries(monkeypatch):
|
||||||
|
waits = []
|
||||||
|
monkeypatch.setattr(dc.time, "sleep", waits.append)
|
||||||
|
routes = {"/users/@me": [
|
||||||
|
_Resp(429, {}, {"Retry-After": "1.5"}), _ok({"username": "me"}),
|
||||||
|
], "/channels/2": _ok({"id": "2", "type": 0})}
|
||||||
|
ok, msg = _client(routes).verify_auth("https://discord.com/channels/1/2")
|
||||||
|
assert ok is True and "me" in msg
|
||||||
|
assert waits == [1.5]
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_tells_a_bad_token_from_a_hidden_channel():
|
||||||
|
bad = _client({"/users/@me": _Resp(401, {})})
|
||||||
|
assert bad.verify_auth("https://discord.com/channels/1/2")[0] is False
|
||||||
|
hidden = _client({"/users/@me": _ok({"username": "me"}),
|
||||||
|
"/channels/2": _Resp(403, {})})
|
||||||
|
ok, msg = hidden.verify_auth("https://discord.com/channels/1/2")
|
||||||
|
assert ok is False and "cannot see this channel" in msg
|
||||||
|
assert _client({}).verify_auth("https://discord.com/channels/1/2/3")[0] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_request_profile_is_gallery_dls():
|
||||||
|
client = _client({})
|
||||||
|
h = client._session.headers
|
||||||
|
assert h["Authorization"] == "tok"
|
||||||
|
assert h["Referer"] == "https://discord.com/"
|
||||||
|
assert h["Accept"] == "*/*"
|
||||||
|
assert h["User-Agent"].startswith("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:")
|
||||||
|
|
||||||
|
|
||||||
|
# -- the adapter ---------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path):
|
||||||
|
from backend.app.services.discord_ingester import DiscordIngester, _ledger_key
|
||||||
|
|
||||||
|
ing = DiscordIngester(tmp_path, None, session_factory=None, auth_token="tok")
|
||||||
|
assert ing.client._session.headers["Authorization"] == "tok"
|
||||||
|
# A files-only drop is ordinary on Discord, not a broken parser.
|
||||||
|
assert ing._body_canary is False
|
||||||
|
[media] = DiscordClient.extract_media(
|
||||||
|
_msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
|
||||||
|
)
|
||||||
|
assert _ledger_key(media) == "9:300"
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"""The native Discord downloader lands files where gallery-dl did (#4414).
|
||||||
|
|
||||||
|
A cutover that names one file differently re-downloads it and imports a
|
||||||
|
duplicate, so these pin the path, the name cleaning and the sidecar pairing
|
||||||
|
against what gallery-dl's config produces — and that the records it writes read
|
||||||
|
back through `parse_sidecar` as the same post the gallery-dl sidecars made.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from backend.app.services.discord_client import DiscordClient
|
||||||
|
from backend.app.services.discord_downloader import (
|
||||||
|
DiscordDownloader,
|
||||||
|
channel_dir,
|
||||||
|
gdl_clean,
|
||||||
|
media_stem,
|
||||||
|
)
|
||||||
|
from backend.app.utils.sidecar import find_sidecar, parse_sidecar
|
||||||
|
|
||||||
|
_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
status_code = 200
|
||||||
|
headers: dict = {}
|
||||||
|
|
||||||
|
def raise_for_status(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def iter_content(self, chunk_size=None):
|
||||||
|
yield _PNG
|
||||||
|
|
||||||
|
|
||||||
|
class _Media:
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def get(self, url, stream=None, timeout=None, headers=None):
|
||||||
|
self.calls.append(url)
|
||||||
|
return _Resp()
|
||||||
|
|
||||||
|
|
||||||
|
def _message(**extra):
|
||||||
|
return {
|
||||||
|
"id": "1234", "type": 0, "timestamp": "2026-09-20T23:30:00.000000+00:00",
|
||||||
|
"content": "new set!", "channel_id": "22",
|
||||||
|
"author": {"id": "7", "username": "artist"},
|
||||||
|
"attachments": [{"url": "https://cdn.discordapp.com/attachments/2/3/Red%3AAlt.PNG?ex=1"}],
|
||||||
|
"embeds": [],
|
||||||
|
"_meta": {"server": "Studio", "server_id": "11", "channel": "drops/nsfw",
|
||||||
|
"channel_id": "22", "is_thread": False},
|
||||||
|
**extra,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _downloader(tmp_path, session=None):
|
||||||
|
return DiscordDownloader(tmp_path, validate=False, session=session or _Media())
|
||||||
|
|
||||||
|
|
||||||
|
def test_names_follow_gallery_dls_pattern_and_linux_cleaning(tmp_path):
|
||||||
|
msg = _message()
|
||||||
|
[media] = DiscordClient.extract_media(msg)
|
||||||
|
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord" / "drops_nsfw"
|
||||||
|
# `:` survives: gallery-dl on Linux only replaces `/`.
|
||||||
|
assert media_stem(msg, media) == "20260920_1234_01_Red:Alt"
|
||||||
|
assert gdl_clean("a\x07b/c") == "ab_c"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_with_no_name_adds_no_directory(tmp_path):
|
||||||
|
msg = _message(_meta={"channel": " "})
|
||||||
|
assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_gallery_dl_already_wrote_is_not_fetched_again(tmp_path):
|
||||||
|
msg = _message()
|
||||||
|
media = DiscordClient.extract_media(msg)
|
||||||
|
existing = tmp_path / "art" / "discord" / "drops_nsfw" / "20260920_1234_01_Red:Alt.png"
|
||||||
|
existing.parent.mkdir(parents=True)
|
||||||
|
existing.write_bytes(_PNG)
|
||||||
|
session = _Media()
|
||||||
|
[out] = _downloader(tmp_path, session).download_post(msg, media, "art")
|
||||||
|
assert out.status == "skipped_disk" and out.path == existing
|
||||||
|
assert session.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_new_file_gets_a_sidecar_the_importer_pairs_to_its_message(tmp_path):
|
||||||
|
msg = _message()
|
||||||
|
[out] = _downloader(tmp_path).download_post(msg, DiscordClient.extract_media(msg), "art")
|
||||||
|
assert out.status == "downloaded"
|
||||||
|
assert out.path.name == "20260920_1234_01_Red:Alt.png"
|
||||||
|
sidecar = find_sidecar(out.path)
|
||||||
|
assert sidecar is not None
|
||||||
|
data = json.loads(sidecar.read_text())
|
||||||
|
# No `id`/`post_id`: either would outrank message_id as the post id.
|
||||||
|
assert "id" not in data and "post_id" not in data
|
||||||
|
sd = parse_sidecar(data)
|
||||||
|
assert sd.external_post_id == "1234"
|
||||||
|
assert sd.source_url.startswith("https://cdn.discordapp.com/")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_seen_file_is_skipped_without_a_request(tmp_path):
|
||||||
|
msg = _message()
|
||||||
|
session = _Media()
|
||||||
|
[out] = _downloader(tmp_path, session).download_post(
|
||||||
|
msg, DiscordClient.extract_media(msg), "art", is_seen=lambda m: True,
|
||||||
|
)
|
||||||
|
assert out.status == "skipped_seen" and session.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_message_record_reads_back_as_the_gallery_dl_post(tmp_path):
|
||||||
|
rec = _downloader(tmp_path).write_post_record(_message(), "art")
|
||||||
|
assert rec.path.name == "20260920_1234_post.json"
|
||||||
|
assert rec.body_chars == len("new set!")
|
||||||
|
sd = parse_sidecar(json.loads(rec.path.read_text()))
|
||||||
|
assert sd.platform == "discord"
|
||||||
|
assert sd.external_post_id == "1234"
|
||||||
|
assert sd.post_url == "https://discord.com/channels/11/22/1234"
|
||||||
|
assert sd.description == "new set!"
|
||||||
|
assert sd.post_date.isoformat().startswith("2026-09-20T23:30")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_record_is_not_a_media_sidecar(tmp_path):
|
||||||
|
"""It must not pair with any file of the message."""
|
||||||
|
msg = _message()
|
||||||
|
dl = _downloader(tmp_path)
|
||||||
|
[out] = dl.download_post(msg, DiscordClient.extract_media(msg), "art")
|
||||||
|
rec = dl.write_post_record(msg, "art")
|
||||||
|
assert find_sidecar(out.path) != rec.path
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_re_read_never_blanks_a_stored_body(tmp_path):
|
||||||
|
rec = _downloader(tmp_path).write_post_record(_message(content=""), "art", revisit=True)
|
||||||
|
assert rec.path is None
|
||||||
@@ -17,7 +17,7 @@ from backend.app.services.gallery_dl import ErrorType
|
|||||||
|
|
||||||
|
|
||||||
def test_native_platforms():
|
def test_native_platforms():
|
||||||
for platform in ("patreon", "subscribestar"):
|
for platform in ("patreon", "subscribestar", "discord"):
|
||||||
assert uses_native_ingester(platform) is True
|
assert uses_native_ingester(platform) is True
|
||||||
assert platform in NATIVE_INGESTER_PLATFORMS
|
assert platform in NATIVE_INGESTER_PLATFORMS
|
||||||
|
|
||||||
@@ -104,8 +104,20 @@ async def test_verifying_a_retired_platform_is_inconclusive_not_rejected():
|
|||||||
def test_gallery_dl_platforms_are_not_native():
|
def test_gallery_dl_platforms_are_not_native():
|
||||||
# The platforms still served by gallery-dl must NOT route to the native
|
# The platforms still served by gallery-dl must NOT route to the native
|
||||||
# ingester — guards an accidental over-broad migration.
|
# ingester — guards an accidental over-broad migration.
|
||||||
for platform in ("hentaifoundry", "discord"):
|
assert uses_native_ingester("hentaifoundry") is False
|
||||||
assert uses_native_ingester(platform) is False
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_discord_verify_without_a_token_is_a_rejection_not_a_request():
|
||||||
|
"""Discord authenticates by token (milestone 428); with none saved there is
|
||||||
|
nothing to send, and saying so beats an HTTP 401 from Discord."""
|
||||||
|
ok, message = await verify_source_credential(
|
||||||
|
platform="discord", url="https://discord.com/channels/1/2",
|
||||||
|
artist_slug="someone", config_overrides=None, cookies_path=None,
|
||||||
|
auth_token=None, images_root=Path("/nonexistent"),
|
||||||
|
)
|
||||||
|
assert ok is False
|
||||||
|
assert "token" in message.lower()
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_platform_is_not_native():
|
def test_unknown_platform_is_not_native():
|
||||||
|
|||||||
@@ -319,6 +319,64 @@ async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
|
|||||||
assert client.consumed_posts == 2
|
assert client.consumed_posts == 2
|
||||||
|
|
||||||
|
|
||||||
|
class _FeedsClient(_FakeClient):
|
||||||
|
"""A source that is several feeds walked in turn (a Discord server's
|
||||||
|
channels), with the optional `skip_feed` seam (#4413). `feeds` is a list of
|
||||||
|
`pages` lists, one per feed."""
|
||||||
|
|
||||||
|
def __init__(self, feeds):
|
||||||
|
super().__init__([page for pages in feeds for page in pages])
|
||||||
|
self._feeds = feeds
|
||||||
|
self._skip = False
|
||||||
|
self.skips = 0
|
||||||
|
|
||||||
|
def skip_feed(self):
|
||||||
|
self._skip = True
|
||||||
|
self.skips += 1
|
||||||
|
|
||||||
|
def iter_posts(self, campaign_id, cursor=None):
|
||||||
|
for pages in self._feeds:
|
||||||
|
self._skip = False
|
||||||
|
self._pages = pages
|
||||||
|
for item in super().iter_posts(campaign_id, cursor):
|
||||||
|
yield item
|
||||||
|
if self._skip:
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_seen_media(sync_engine, source_id, media):
|
||||||
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||||
|
with factory() as s:
|
||||||
|
for m in media:
|
||||||
|
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
|
||||||
|
s.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_quiet_feed_ends_itself_not_the_walk(source_id, sync_engine, tmp_path):
|
||||||
|
"""#4413: a Discord server's first channel is all seen; the tick must still
|
||||||
|
reach the second channel's new file, instead of stopping at the first."""
|
||||||
|
quiet = [_media(f"a{i}", 1) for i in range(1, 5)]
|
||||||
|
_seed_seen_media(sync_engine, source_id, quiet)
|
||||||
|
fresh = _media("b1", 1)
|
||||||
|
client = _FeedsClient([
|
||||||
|
[(None, [(m.post_id, [m]) for m in quiet])],
|
||||||
|
[(None, [("b1", [fresh])])],
|
||||||
|
])
|
||||||
|
downloader = _FakeDownloader(tmp_path)
|
||||||
|
result = _ingester(sync_engine, tmp_path, client, downloader).run(
|
||||||
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||||
|
url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
|
||||||
|
)
|
||||||
|
assert result.success is True
|
||||||
|
assert client.skips == 1
|
||||||
|
# The quiet feed stopped after its 2nd seen post; the next feed was walked.
|
||||||
|
assert client.consumed_posts == 3
|
||||||
|
assert downloader.download_calls == 1
|
||||||
|
assert "1 feed(s) caught up" in result.stdout
|
||||||
|
assert "reached end" not in result.stdout
|
||||||
|
|
||||||
|
|
||||||
# --- backfill -------------------------------------------------------------
|
# --- backfill -------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,11 @@ pytestmark = pytest.mark.integration
|
|||||||
def test_non_serialized_platform_has_no_lock():
|
def test_non_serialized_platform_has_no_lock():
|
||||||
# gallery-dl platforms aren't capped — they get no lock at all.
|
# gallery-dl platforms aren't capped — they get no lock at all.
|
||||||
assert platform_lock("hentaifoundry", ttl_seconds=60) is None
|
assert platform_lock("hentaifoundry", ttl_seconds=60) is None
|
||||||
assert platform_lock("discord", ttl_seconds=60) is None
|
|
||||||
|
|
||||||
|
def test_discord_is_serialized():
|
||||||
|
# Native since milestone 428, and every source shares one user token.
|
||||||
|
assert platform_lock("discord", ttl_seconds=60) is not None
|
||||||
|
|
||||||
|
|
||||||
def test_subscribestar_is_serialized():
|
def test_subscribestar_is_serialized():
|
||||||
|
|||||||
Reference in New Issue
Block a user