pixiv retired, placement reconciler removed, build race fixed, mirror pinned #257
@@ -0,0 +1,138 @@
|
||||
"""Drop pixiv's ledgers, and delete credentials for platforms FC no longer has.
|
||||
|
||||
Milestone #406 step 6 (with issue #3980 folded in). Phase 1 unregistered pixiv
|
||||
and the commit alongside this one deleted its client, downloader, ingester and
|
||||
models. This removes the data those models described, and the stored secrets of
|
||||
every platform that has been retired.
|
||||
|
||||
## The two ledger tables
|
||||
|
||||
`pixiv_seen_media` and `pixiv_failed_media` are the per-source seen / dead-letter
|
||||
ledgers for a downloader that no longer exists. They were created in
|
||||
`0089_baseline.py`, so dropping them needs a new revision rather than an edit
|
||||
there.
|
||||
|
||||
## The credentials
|
||||
|
||||
Written as *delete every credential whose platform is not registered* rather
|
||||
than as `platform = 'pixiv'`, at the explicit ask in this step's plan. That is
|
||||
what makes one migration cover two retirements:
|
||||
|
||||
- **pixiv** — a live OAuth refresh token for a service FC no longer talks to.
|
||||
- **deviantart** — issue #3980. #3069 retired DeviantArt in code on 2026-08-27
|
||||
and left its stored session behind; seven weeks later it was still there.
|
||||
|
||||
And it is the only way either row can go. The credentials UI
|
||||
(`subscriptions/SettingsTab.vue`) renders one card per platform returned by
|
||||
`/api/platforms`, then looks the credential up by key — so a row whose platform
|
||||
is unregistered has no card, no Remove button, and no way for the operator to
|
||||
reach it. `CredentialService.list()` would return it; nothing asks.
|
||||
|
||||
The registered set is written out literally instead of importing
|
||||
`known_platform_keys()`. A migration is a statement about one moment in the
|
||||
schema's history: if it imported the live registry, retiring a fifth platform
|
||||
in 2027 would silently change what this 2026 revision did on a fresh database.
|
||||
The list below is the registry as of 2026-09-21.
|
||||
|
||||
## What is deliberately kept
|
||||
|
||||
**Every pixiv `Source` row.** The original plan deleted them; the operator's
|
||||
call on 2026-09-21 was to keep them, and the reason is that `platform` is
|
||||
stored ONLY on `Source` — neither `Post` nor `ImageRecord` carries it. Both
|
||||
FKs are `ON DELETE SET NULL`, so a delete would not lose the art, but it would
|
||||
drop every pixiv image into the gallery's `__unsourced__` bucket and strip the
|
||||
platform chip off every pixiv post. The rows stay disabled (0097) and their
|
||||
platform is unregistered, so nothing schedules them, nothing downloads through
|
||||
them, and `POST /api/sources` will not make another. Keeping them costs
|
||||
nothing and keeps the attribution the milestone's goal — *"the art already
|
||||
downloaded from pixiv stays"* — is actually about.
|
||||
|
||||
Every pixiv `Post` and `ImageRecord` is likewise untouched.
|
||||
|
||||
Revision ID: 0102
|
||||
Revises: 0101
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0102"
|
||||
down_revision: Union[str, None] = "0101"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# services/platforms/__init__.py's PLATFORMS as of this revision. See the
|
||||
# docstring for why this is a literal and not an import.
|
||||
_REGISTERED_PLATFORMS = ("patreon", "subscribestar", "hentaifoundry", "discord")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
sa.text(
|
||||
"DELETE FROM credential WHERE platform NOT IN :registered"
|
||||
).bindparams(
|
||||
sa.bindparam("registered", value=_REGISTERED_PLATFORMS, expanding=True)
|
||||
)
|
||||
)
|
||||
op.drop_index("ix_pixiv_failed_media_source_id", table_name="pixiv_failed_media")
|
||||
op.drop_table("pixiv_failed_media")
|
||||
op.drop_index("ix_pixiv_seen_media_source_id", table_name="pixiv_seen_media")
|
||||
op.drop_table("pixiv_seen_media")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The tables come back empty, and the credentials do not come back at all:
|
||||
# they were encrypted blobs, this migration does not copy them anywhere,
|
||||
# and restoring a live token for a platform FC cannot talk to would only
|
||||
# re-create the liability. Rule #22 owes no story backwards.
|
||||
op.create_table(
|
||||
"pixiv_seen_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_pixiv_seen_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_pixiv_seen_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_seen_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_pixiv_seen_media_source_id"), "pixiv_seen_media", ["source_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"pixiv_failed_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_pixiv_failed_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_pixiv_failed_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_failed_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_pixiv_failed_media_source_id"), "pixiv_failed_media", ["source_id"],
|
||||
unique=False,
|
||||
)
|
||||
@@ -109,8 +109,8 @@ async def quick_add_source():
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
# crypto lets a pixiv add resolve the artist's display name via the
|
||||
# stored OAuth token (else it falls back to the numeric id). #130.
|
||||
# crypto lets an add resolve the artist's display name via the
|
||||
# stored credential (else it falls back to the URL handle). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
|
||||
@@ -26,8 +26,6 @@ from .membership_sync import MembershipSync
|
||||
from .ml_settings import MLSettings
|
||||
from .patreon_failed_media import PatreonFailedMedia
|
||||
from .patreon_seen_media import PatreonSeenMedia
|
||||
from .pixiv_failed_media import PixivFailedMedia
|
||||
from .pixiv_seen_media import PixivSeenMedia
|
||||
from .platform_membership import PlatformMembership
|
||||
from .post import Post
|
||||
from .post_association import PostAssociation
|
||||
@@ -58,8 +56,6 @@ __all__ = [
|
||||
"Credential",
|
||||
"PatreonFailedMedia",
|
||||
"PatreonSeenMedia",
|
||||
"PixivFailedMedia",
|
||||
"PixivSeenMedia",
|
||||
"SubscribeStarFailedMedia",
|
||||
"SubscribeStarSeenMedia",
|
||||
"Post",
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
|
||||
failing to download/validate.
|
||||
|
||||
Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
|
||||
walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
|
||||
otherwise re-error forever and re-burn backfill chunks. After ``attempts``
|
||||
reaches the dead-letter threshold the ingester skips it on routine
|
||||
tick/backfill walks (recovery still re-attempts). A later clean download
|
||||
clears the row.
|
||||
|
||||
`filehash` is the same synthesized ``<illust_id>:p<num>`` /
|
||||
``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
|
||||
is the upsert key.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PixivFailedMedia(Base):
|
||||
__tablename__ = "pixiv_failed_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_failed_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
last_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -1,42 +0,0 @@
|
||||
"""PixivSeenMedia — per-source ledger of Pixiv media already
|
||||
downloaded+processed.
|
||||
|
||||
Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
|
||||
ingester (replacing gallery-dl). One queryable row per (source, media) so
|
||||
routine walks skip media we've already ingested; recovery mode bypasses the
|
||||
ledger to re-walk.
|
||||
|
||||
Pixiv original URLs carry no content hash, so `filehash` is always the
|
||||
synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
|
||||
zip) key — stable across any URL-shape drift. String(128) matches the sibling
|
||||
ledgers.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PixivSeenMedia(Base):
|
||||
__tablename__ = "pixiv_seen_media"
|
||||
__table_args__ = (
|
||||
# Dedup key the downloader upserts against: one ledger row per
|
||||
# (source, media). A second sighting of the same media is a no-op.
|
||||
UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_seen_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -26,14 +26,12 @@ from pathlib import Path
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
from .pixiv_client import user_id_from_url
|
||||
from .pixiv_ingester import PixivIngester
|
||||
from .platforms import known_platform_keys
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
|
||||
# they migrate too. pixiv left this set when it was retired (milestone #406).
|
||||
# they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
|
||||
|
||||
@@ -67,7 +65,6 @@ def _native_ingester_cls(platform: str):
|
||||
dispatch pick up the replacement."""
|
||||
return {
|
||||
"patreon": PatreonIngester,
|
||||
"pixiv": PixivIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
}[platform]
|
||||
|
||||
@@ -127,25 +124,17 @@ async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
|
||||
from it (no lookup → resolved None either way). Patreon resolves the
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
return url, None
|
||||
if platform == "pixiv":
|
||||
return user_id_from_url(url), None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
|
||||
def _campaign_resolution_error(platform: str, url: str) -> str:
|
||||
"""Operator-facing message for a native source whose campaign id could not
|
||||
be resolved — names the platform's own lookup mechanism."""
|
||||
if platform == "pixiv":
|
||||
return (
|
||||
f"Could not extract a pixiv user id. source_url={url!r} — expected "
|
||||
"a URL like https://www.pixiv.net/users/<id>."
|
||||
)
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
@@ -172,8 +161,8 @@ async def _run_native_ingester(
|
||||
platform, ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
if not campaign_id:
|
||||
# Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
|
||||
# (SubscribeStar's campaign id is the URL itself — never lands here.)
|
||||
# Patreon: vanity lookup failed. (SubscribeStar's campaign id is the
|
||||
# URL itself — never lands here.)
|
||||
url = ctx["url"]
|
||||
return (
|
||||
DownloadResult(
|
||||
@@ -205,7 +194,7 @@ async def _run_native_ingester(
|
||||
validate=gdl._validate_files,
|
||||
rate_limit=rate_limit,
|
||||
request_sleep=request_sleep,
|
||||
# Uniform across adapters: token platforms (pixiv) authenticate with
|
||||
# Uniform across adapters: a token platform would authenticate with
|
||||
# it, cookie platforms accept-and-ignore — so this construction stays
|
||||
# platform-agnostic.
|
||||
auth_token=ctx["auth_token"],
|
||||
@@ -252,16 +241,11 @@ async def verify_source_credential(
|
||||
if uses_native_ingester(platform):
|
||||
# Native ingester platforms verify via their own lightweight auth probe.
|
||||
# SubscribeStar's probe takes the creator URL directly; Patreon's
|
||||
# resolves the campaign id first; Pixiv's is one OAuth refresh (the
|
||||
# exact call that fails when the token is bad — no feed walk).
|
||||
# resolves the campaign id first.
|
||||
if platform == "subscribestar":
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
if platform == "pixiv":
|
||||
from .pixiv_ingester import verify_pixiv_credential
|
||||
|
||||
return await verify_pixiv_credential(auth_token)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
@@ -61,8 +61,8 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
class ExtensionService:
|
||||
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
||||
self.session = session
|
||||
# Optional decryptor for resolving a token-auth platform's display name
|
||||
# (pixiv) at add-time. None → skip resolution, fall back to the handle.
|
||||
# Optional decryptor for resolving a platform's display name at
|
||||
# add-time. None → skip resolution, fall back to the handle.
|
||||
self._crypto = crypto
|
||||
|
||||
async def quick_add_source(self, url: str) -> dict:
|
||||
@@ -112,12 +112,12 @@ class ExtensionService:
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str:
|
||||
"""The real display name for a new artist, resolved from the platform at
|
||||
add-time (#130). Our native platforms each have a name source — pixiv the
|
||||
app API (token), patreon the campaigns API, subscribestar the profile
|
||||
page (both cookies). Other platforms (and any failure — no credential,
|
||||
network error) fall back to the URL handle, which is already readable.
|
||||
add-time (#130). Our native platforms each have a name source — patreon
|
||||
the campaigns API, subscribestar the profile page (both cookies). Other
|
||||
platforms (and any failure — no credential, network error) fall back to
|
||||
the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
||||
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
import asyncio
|
||||
|
||||
@@ -125,15 +125,7 @@ class ExtensionService:
|
||||
cred = CredentialService(self.session, self._crypto)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
if platform == "pixiv":
|
||||
token = await cred.get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
from .pixiv_client import PixivClient
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
elif platform == "patreon":
|
||||
if platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
|
||||
@@ -45,7 +45,7 @@ from .tag_query import (
|
||||
# provenance (filesystem imports). Returned by facets() as a null-valued
|
||||
# bucket; the frontend maps that null back to this sentinel in the URL so the
|
||||
# bucket is selectable. Underscore-wrapped so it can't collide with a real
|
||||
# gallery-dl platform name (patreon/pixiv/...).
|
||||
# gallery-dl platform name (patreon/hentaifoundry/...).
|
||||
UNSOURCED_PLATFORM = "__unsourced__"
|
||||
|
||||
|
||||
|
||||
@@ -113,9 +113,9 @@ class Ingester:
|
||||
# (e.g. "Patreon API", "SubscribeStar markup").
|
||||
self._drift_label = drift_label or platform
|
||||
# #862 canary opt-out: platforms whose posts legitimately have empty
|
||||
# bodies across large samples (pixiv — caption-less artists are common)
|
||||
# would false-positive the zero-bodies-means-drift alarm; their clients
|
||||
# catch drift structurally (response-shape checks) instead. The
|
||||
# bodies across large samples would false-positive the
|
||||
# zero-bodies-means-drift alarm; their clients catch drift structurally
|
||||
# (response-shape checks) instead. The
|
||||
# "bodies X/N" summary line still surfaces the ratio either way.
|
||||
self._body_canary = body_canary
|
||||
|
||||
|
||||
@@ -1,579 +0,0 @@
|
||||
"""Native Pixiv client — the Pixiv adapter's read path.
|
||||
|
||||
Pixiv has a real (if unofficial) API: the mobile app API gallery-dl drives
|
||||
(`PixivAppAPI`). Per the downloader ground rule — gallery-dl is the
|
||||
known-working base — this client mirrors gallery-dl 1.32.5's request profile
|
||||
EXACTLY: the same iOS app headers on every request, the same OAuth
|
||||
refresh-token dance against oauth.secure.pixiv.net (X-Client-Time +
|
||||
X-Client-Hash), and the same `/v1/user/illusts` walk paginated by `next_url`.
|
||||
Deviating from that profile is how the SubscribeStar/Patreon spikes broke, so
|
||||
any change here should be diffed against gallery-dl's extractor first.
|
||||
|
||||
Feed shape (characterized from gallery-dl 1.32.5, extractor/pixiv.py):
|
||||
- `GET /v1/user/illusts?user_id=<id>` returns `{"illusts": [work...],
|
||||
"next_url": "https://app-api...?user_id=..&offset=30" | null}`.
|
||||
- Pagination: re-issue the SAME endpoint with `next_url`'s query params. The
|
||||
query string doubles as our resumable page cursor (re-fetching it re-serves
|
||||
the same page — the ingest-core resume contract).
|
||||
- A work carries id/title/type(illust|manga|ugoira)/caption(HTML)/
|
||||
create_date(ISO+09:00)/tags[{name,translated_name}]/user/page_count/
|
||||
x_restrict/series/total_view/total_bookmarks/meta_single_page/meta_pages.
|
||||
- Files: multi-page → meta_pages[].image_urls.original; single page →
|
||||
meta_single_page.original_image_url; ugoira → `/v1/ugoira/metadata` zip
|
||||
(600x600 → 1920x1080 URL swap, gallery-dl's default non-original mode).
|
||||
|
||||
`campaign_id` for Pixiv is the numeric user id (extracted from the source URL
|
||||
by `user_id_from_url` — no network resolver needed).
|
||||
|
||||
Gated works: pixiv serves a `https://s.pximg.net/common/images/limit_*.png`
|
||||
placeholder as the "original" when a work is blocked for this account
|
||||
(sanity-level filter, my-pixiv lock, deleted). gallery-dl's fallback for those
|
||||
is a web-AJAX scrape that needs PHPSESSID browser cookies — FC stores only the
|
||||
OAuth refresh token, so (exactly like our previous gallery-dl configuration,
|
||||
which warned "No PHPSESSID cookie set") those works are skipped, via the
|
||||
post_is_gated seam. Auth failures are loud (rotate the refresh token); a
|
||||
response missing the fields we depend on is DRIFT (update this client).
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import safe_ext
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
NativeIngestError,
|
||||
make_session,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TIMEOUT_SECONDS = 30.0
|
||||
_API_ROOT = "https://app-api.pixiv.net"
|
||||
_OAUTH_URL = "https://oauth.secure.pixiv.net/auth/token"
|
||||
|
||||
# gallery-dl's public Pixiv-app credentials (PixivAppAPI, also pixivpy's) —
|
||||
# these identify the official iOS app to the API, NOT the operator; the
|
||||
# operator's identity is the OAuth refresh token.
|
||||
_CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
|
||||
_CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
|
||||
_HASH_SECRET = (
|
||||
"28c1fdd170a5204386cb1313c7077b34"
|
||||
"f83e4aaf4aa829ce78c231e05b0bae2c"
|
||||
)
|
||||
|
||||
# The exact header set gallery-dl 1.32.5 installs on its session — the proven
|
||||
# app-API request profile. The Referer also unlocks i.pximg.net media GETs
|
||||
# (403 without it), so the downloader reuses this constant.
|
||||
PIXIV_APP_HEADERS = {
|
||||
"App-OS": "ios",
|
||||
"App-OS-Version": "16.7.2",
|
||||
"App-Version": "7.19.1",
|
||||
"User-Agent": "PixivIOSApp/7.19.1 (iOS 16.7.2; iPhone12,8)",
|
||||
"Referer": "https://app-api.pixiv.net/",
|
||||
}
|
||||
|
||||
# Placeholder image prefix pixiv serves instead of a blocked work's original
|
||||
# (limit_sanity_level / limit_mypixiv / limit_unknown variants).
|
||||
_LIMIT_URL = "https://s.pximg.net/common/images/limit_"
|
||||
|
||||
# The app API reports rate-limiting as an error MESSAGE (often on HTTP 403),
|
||||
# not only as HTTP 429. gallery-dl sleeps 300s in-walk; sleeping that long
|
||||
# inside our time-boxed chunk would eat the whole budget, so we surface it as
|
||||
# a typed 429 and let download_service's cooldown machinery honor the wait.
|
||||
_RATE_LIMIT_RETRY_AFTER = 300.0
|
||||
|
||||
_TITLE_MAX = 50 # gallery-dl pixiv filename template: {title[:50]}
|
||||
|
||||
_RATINGS = {0: "General", 1: "R-18", 2: "R-18G"}
|
||||
|
||||
|
||||
class PixivAPIError(NativeIngestError):
|
||||
"""Base for native Pixiv client failures. status_code / retry_after are
|
||||
inherited from NativeIngestError."""
|
||||
|
||||
|
||||
class PixivAuthError(PixivAPIError, NativeAuthError):
|
||||
"""Auth failure — missing/expired/revoked OAuth refresh token. Fix =
|
||||
rotate the credential (Settings → Credentials → Pixiv), not update the
|
||||
client. Maps to error_type 'auth_error'."""
|
||||
|
||||
|
||||
class PixivDriftError(PixivAPIError, NativeDriftError):
|
||||
"""A response did not match the shape this client depends on (missing
|
||||
`illusts`, un-parseable JSON where JSON was promised). Fail loud so the
|
||||
run flags 'the Pixiv app API changed' instead of silently importing
|
||||
nothing. Maps to API_DRIFT."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable file belonging to a Pixiv work.
|
||||
|
||||
Fields mirror the other native clients' MediaItem so the downloader and
|
||||
ledger are structurally the same. Pixiv original URLs carry no content
|
||||
hash, so `filehash` is always None and the ledger keys on
|
||||
`<post_id>:<media_id>` where media_id is `p<num>` (page) or `ugoira`
|
||||
(the frame zip) — stable across URL-shape drift.
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
filehash: str | None
|
||||
post_id: str
|
||||
media_id: str
|
||||
|
||||
|
||||
def user_id_from_url(url: str) -> str | None:
|
||||
"""The numeric pixiv user id from a source URL, or None.
|
||||
|
||||
Handles the modern forms FC accepts as sources
|
||||
(https://www.pixiv.net/users/<id>, /en/users/<id>) plus the legacy
|
||||
member.php?id=<id>. This IS the campaign id — no network resolver.
|
||||
"""
|
||||
parts = urlsplit(url or "")
|
||||
if "pixiv.net" not in parts.netloc:
|
||||
return None
|
||||
segs = [s for s in parts.path.split("/") if s]
|
||||
if segs and segs[0] == "en":
|
||||
segs = segs[1:]
|
||||
if len(segs) >= 2 and segs[0] == "users" and segs[1].isdigit():
|
||||
return segs[1]
|
||||
if segs and segs[0] == "member.php":
|
||||
qid = dict(parse_qsl(parts.query)).get("id", "")
|
||||
if qid.isdigit():
|
||||
return qid
|
||||
return None
|
||||
|
||||
|
||||
def _work_filename(work: dict, num: int, url: str) -> str:
|
||||
"""gallery-dl layout parity: `{id}_{title[:50]}_{num:>02}.{extension}`
|
||||
(the downloader sanitizes the final segment)."""
|
||||
title = work.get("title")
|
||||
title50 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
|
||||
ext = safe_ext(urlsplit(url).path.rsplit("/", 1)[-1])
|
||||
return f"{work.get('id')}_{title50}_{num:02d}{ext}"
|
||||
|
||||
|
||||
class PixivClient:
|
||||
"""Synchronous Pixiv app-API read client. Construct with the operator's
|
||||
OAuth refresh token (the same token-type Credential the gallery-dl path
|
||||
consumed as `extractor.pixiv.refresh-token`)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
refresh_token: str | None,
|
||||
*,
|
||||
request_sleep: float = 0.0,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self.refresh_token = refresh_token
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
self._max_retries = max_retries
|
||||
# No cookies — the app API authenticates via the Bearer token _login
|
||||
# installs. make_session still supplies the retry/UA plumbing; the
|
||||
# extra_headers overwrite its browser UA with the app profile.
|
||||
self._session = (
|
||||
session if session is not None
|
||||
else make_session(None, extra_headers=PIXIV_APP_HEADERS)
|
||||
)
|
||||
self._authed_user: dict = {}
|
||||
# Monotonic deadline after which the access token must be refreshed;
|
||||
# 0 forces a refresh on first use.
|
||||
self._token_deadline = 0.0
|
||||
|
||||
# -- auth ----------------------------------------------------------------
|
||||
|
||||
def _login(self) -> None:
|
||||
"""Exchange the refresh token for a Bearer access token (gallery-dl's
|
||||
`_login_impl`, including the X-Client-Time/X-Client-Hash pair the
|
||||
endpoint validates). No-op while the current token is still fresh."""
|
||||
if time.monotonic() < self._token_deadline:
|
||||
return
|
||||
if not self.refresh_token:
|
||||
raise PixivAuthError(
|
||||
"No Pixiv refresh token configured — add the OAuth refresh "
|
||||
"token as the Pixiv credential (token type)."
|
||||
)
|
||||
# gallery-dl stamps naive-UTC with a literal +00:00 suffix.
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S+00:00")
|
||||
headers = {
|
||||
"X-Client-Time": now,
|
||||
"X-Client-Hash": hashlib.md5(
|
||||
(now + _HASH_SECRET).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
data = {
|
||||
"client_id": _CLIENT_ID,
|
||||
"client_secret": _CLIENT_SECRET,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": self.refresh_token,
|
||||
"get_secure_url": "1",
|
||||
}
|
||||
try:
|
||||
resp = self._session.post(
|
||||
_OAUTH_URL, data=data, headers=headers,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise PixivAPIError(f"Pixiv OAuth request failed: {exc}") from exc
|
||||
if resp.status_code >= 400:
|
||||
raise PixivAuthError(
|
||||
"Pixiv rejected the refresh token (HTTP "
|
||||
f"{resp.status_code}) — rotate the Pixiv credential.",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
try:
|
||||
payload = resp.json()["response"]
|
||||
access = payload["access_token"]
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise PixivDriftError(
|
||||
f"Pixiv OAuth response shape changed: {exc}"
|
||||
) from exc
|
||||
self._authed_user = payload.get("user") or {}
|
||||
self._session.headers["Authorization"] = f"Bearer {access}"
|
||||
# expires_in is 3600 today; refresh 60s early so a long walk never
|
||||
# rides an expiring token into a spurious 400.
|
||||
expires_in = payload.get("expires_in")
|
||||
lifetime = float(expires_in) if isinstance(expires_in, (int, float)) else 3600.0
|
||||
self._token_deadline = time.monotonic() + max(60.0, lifetime - 60.0)
|
||||
|
||||
# -- request -------------------------------------------------------------
|
||||
|
||||
def _call(self, endpoint: str, params: dict) -> dict:
|
||||
"""Authenticated app-API GET → parsed JSON body, with the shared 429
|
||||
backoff and the loud auth/drift/rate-limit mapping."""
|
||||
self._login()
|
||||
if self._request_sleep > 0:
|
||||
time.sleep(self._request_sleep)
|
||||
url = _API_ROOT + endpoint
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
resp = self._session.get(
|
||||
url, params=params, timeout=_TIMEOUT_SECONDS
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise PixivAPIError(
|
||||
f"Pixiv request failed ({endpoint}): {exc}"
|
||||
) from exc
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"Pixiv 429 (%s) — backing off %.1fs (retry %d/%d)",
|
||||
endpoint, delay, attempt, self._max_retries,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError as exc:
|
||||
raise PixivDriftError(
|
||||
f"Pixiv returned non-JSON for {endpoint} "
|
||||
f"(HTTP {resp.status_code})"
|
||||
) from exc
|
||||
|
||||
error = body.get("error") if isinstance(body, dict) else None
|
||||
message = ""
|
||||
if isinstance(error, dict):
|
||||
message = str(
|
||||
error.get("user_message") or error.get("message") or ""
|
||||
)
|
||||
# Rate limiting first: the app API reports it as an error MESSAGE
|
||||
# (often on HTTP 403), which must not be mistaken for an auth failure.
|
||||
if resp.status_code == 429 or "rate limit" in message.lower():
|
||||
raise PixivAPIError(
|
||||
f"Pixiv rate limit hit ({endpoint}): {message or 'HTTP 429'}",
|
||||
status_code=429,
|
||||
retry_after=_RATE_LIMIT_RETRY_AFTER,
|
||||
)
|
||||
if resp.status_code in (400, 401, 403):
|
||||
# Invalid/expired access token surfaces as 400 invalid_grant-style
|
||||
# errors on the app API; 401/403 are straight auth rejections.
|
||||
raise PixivAuthError(
|
||||
f"Pixiv rejected the request ({endpoint}, HTTP "
|
||||
f"{resp.status_code}): {message or 'auth rejected'} — "
|
||||
"rotate the Pixiv refresh token.",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise PixivAPIError(
|
||||
f"Pixiv API error ({endpoint}, HTTP {resp.status_code}): "
|
||||
f"{message or 'unknown error'}",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
if error:
|
||||
# HTTP 200 carrying an error object — unexpected, but never
|
||||
# silently treat it as data.
|
||||
raise PixivAPIError(
|
||||
f"Pixiv API error ({endpoint}): {message or error}",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
return body
|
||||
|
||||
# -- normalization -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _normalize(work: dict) -> dict:
|
||||
"""Wrap an app-API work in the `{"id", "attributes", ...}` post shape
|
||||
the platform-agnostic core and shared helpers read. The raw work rides
|
||||
along under `_work` for extract_media / the post record."""
|
||||
title = work.get("title")
|
||||
caption = work.get("caption")
|
||||
wtype = work.get("type")
|
||||
return {
|
||||
"id": work.get("id"),
|
||||
"attributes": {
|
||||
"title": title if isinstance(title, str) else "",
|
||||
"content": caption if isinstance(caption, str) else "",
|
||||
"published_at": work.get("create_date"),
|
||||
"post_type": wtype if isinstance(wtype, str) else "illust",
|
||||
},
|
||||
"_work": work,
|
||||
}
|
||||
|
||||
# -- post-first seams ----------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def post_record_key(post: dict) -> tuple[str, str] | None:
|
||||
"""`(ledger_key, post_id)` gating post-record capture through the seen
|
||||
ledger (`post:<id>` synthetic key), or None when the work has no id."""
|
||||
pid = post.get("id")
|
||||
pid = str(pid) if pid is not None else ""
|
||||
if not pid:
|
||||
return None
|
||||
return (f"post:{pid}", pid)
|
||||
|
||||
@staticmethod
|
||||
def post_meta(post: dict) -> dict:
|
||||
attrs = post.get("attributes") or {}
|
||||
return {"title": attrs.get("title") or None, "date": attrs.get("published_at")}
|
||||
|
||||
@staticmethod
|
||||
def post_is_gated(post: dict) -> bool:
|
||||
"""True when this account cannot fetch the work's real files: pixiv
|
||||
substitutes a `limit_*` placeholder for the original (sanity-level
|
||||
filter / my-pixiv lock / deleted), or zeroes the author (deleted
|
||||
account). Mirrors #874 semantics: gated content leaves NO trace — a
|
||||
placeholder thumbnail and an empty stub would only pollute the
|
||||
archive. (gallery-dl's PHPSESSID web-scrape fallback for these is out
|
||||
of scope: FC holds no pixiv browser cookies — module docstring.)"""
|
||||
work = post.get("_work") or {}
|
||||
user = work.get("user") or {}
|
||||
if not user.get("id"):
|
||||
return True
|
||||
if work.get("meta_pages"):
|
||||
return False
|
||||
single = work.get("meta_single_page") or {}
|
||||
original = single.get("original_image_url")
|
||||
return isinstance(original, str) and original.startswith(_LIMIT_URL)
|
||||
|
||||
# -- media ---------------------------------------------------------------
|
||||
|
||||
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
||||
"""Resolve a work's downloadable files (gallery-dl's `_extract_files`):
|
||||
multi-page originals, the single-page original, or the ugoira frame
|
||||
zip. `included_index` is unused (pixiv works are self-contained)."""
|
||||
work = post.get("_work") or {}
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid or self.post_is_gated(post):
|
||||
return []
|
||||
|
||||
if work.get("type") == "ugoira":
|
||||
return self._ugoira_media(work, pid)
|
||||
|
||||
meta_pages = work.get("meta_pages") or []
|
||||
if meta_pages:
|
||||
items = []
|
||||
for num, page in enumerate(meta_pages):
|
||||
urls = page.get("image_urls") or {}
|
||||
url = urls.get("original")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, num, url),
|
||||
kind="image",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id=f"p{num}",
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
single = work.get("meta_single_page") or {}
|
||||
url = single.get("original_image_url")
|
||||
if not isinstance(url, str) or not url or url.startswith(_LIMIT_URL):
|
||||
return []
|
||||
return [
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, 0, url),
|
||||
kind="image",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id="p0",
|
||||
)
|
||||
]
|
||||
|
||||
def _ugoira_meta(self, work: dict, pid: str) -> dict | None:
|
||||
"""Fetch + memoize the ugoira metadata (frames + zip urls) for a work.
|
||||
|
||||
Idempotent and cached on the work dict, so the post record and the
|
||||
media extraction share ONE `/v1/ugoira/metadata` call regardless of
|
||||
which runs first (the core writes the post record BEFORE it extracts
|
||||
media). Returns None — and caches the miss — on a non-auth failure
|
||||
(matching gallery-dl's downgrade); auth failures stay loud."""
|
||||
if "_ugoira_meta" in work:
|
||||
return work["_ugoira_meta"]
|
||||
try:
|
||||
body = self._call("/v1/ugoira/metadata", {"illust_id": pid})
|
||||
meta = body["ugoira_metadata"]
|
||||
except PixivAuthError:
|
||||
raise
|
||||
except (PixivAPIError, KeyError, TypeError) as exc:
|
||||
log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc)
|
||||
work["_ugoira_meta"] = None
|
||||
return None
|
||||
work["_ugoira_meta"] = meta
|
||||
# Frame delays: a future ugoira→video conversion needs the timings (the
|
||||
# zip alone has none), so the post record captures them.
|
||||
work["_ugoira_frames"] = meta.get("frames") or []
|
||||
return meta
|
||||
|
||||
def fetch_ugoira_frames(self, post: dict) -> None:
|
||||
"""Populate `post['_work']['_ugoira_frames']` for an ugoira post (no-op
|
||||
otherwise). The core writes the post record BEFORE extract_media, so
|
||||
without this the frame timings would never reach the record; this
|
||||
fetches (and memoizes, so extract_media reuses it) the metadata. Injected
|
||||
into the downloader by the ingester, mirroring Patreon's content_fetcher.
|
||||
Auth errors propagate; other failures leave frames unset."""
|
||||
work = post.get("_work") or {}
|
||||
if work.get("type") != "ugoira":
|
||||
return
|
||||
pid = str(post.get("id") or "")
|
||||
if pid:
|
||||
self._ugoira_meta(work, pid)
|
||||
|
||||
def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]:
|
||||
"""The ugoira frame zip (gallery-dl's default non-original mode):
|
||||
`/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080
|
||||
swap. A metadata failure downgrades to 'no media' with a warning
|
||||
(matching gallery-dl) instead of failing the walk — except auth
|
||||
failures, which stay loud."""
|
||||
meta = self._ugoira_meta(work, pid)
|
||||
if meta is None:
|
||||
return []
|
||||
try:
|
||||
zip_url = meta["zip_urls"]["medium"]
|
||||
except (KeyError, TypeError) as exc:
|
||||
log.warning("Pixiv ugoira zip url missing for %s: %s", pid, exc)
|
||||
return []
|
||||
url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1)
|
||||
return [
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, 0, url),
|
||||
kind="ugoira",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id="ugoira",
|
||||
)
|
||||
]
|
||||
|
||||
# -- iteration -----------------------------------------------------------
|
||||
|
||||
def iter_posts(
|
||||
self, campaign_id: str, cursor: str | None = None
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
"""Yield (post, {}, page_cursor) for every work in the user's feed.
|
||||
|
||||
`campaign_id` is the numeric pixiv user id. `cursor` is the query
|
||||
string of the app API's `next_url` (offset pagination); None fetches
|
||||
page 1. The yielded `page_cursor` is the cursor that FETCHED this
|
||||
work's page, so the core checkpoints a value that re-serves the same
|
||||
page on resume (the shared cursor contract)."""
|
||||
if not str(campaign_id or "").isdigit():
|
||||
raise PixivDriftError(
|
||||
f"Pixiv campaign id must be a numeric user id, got "
|
||||
f"{campaign_id!r}"
|
||||
)
|
||||
current = cursor
|
||||
while True:
|
||||
page_cursor = current
|
||||
if current is None:
|
||||
params: dict = {"user_id": campaign_id}
|
||||
else:
|
||||
params = dict(parse_qsl(current))
|
||||
data = self._call("/v1/user/illusts", params)
|
||||
works = data.get("illusts")
|
||||
if not isinstance(works, list):
|
||||
raise PixivDriftError(
|
||||
"Pixiv user-illusts response had no 'illusts' list "
|
||||
f"(keys: {sorted(data)[:8]})"
|
||||
)
|
||||
for work in works:
|
||||
if not isinstance(work, dict):
|
||||
continue
|
||||
yield self._normalize(work), {}, page_cursor
|
||||
next_url = data.get("next_url")
|
||||
if not next_url:
|
||||
return
|
||||
current = str(next_url).rpartition("?")[2]
|
||||
|
||||
# -- user detail ---------------------------------------------------------
|
||||
|
||||
def resolve_display_name(self, user_id: str) -> str | None:
|
||||
"""The pixiv user's display name via `/v1/user/detail` (gallery-dl's
|
||||
user_detail) — used to name the Artist when a source is added by numeric
|
||||
id. None on any failure (the caller falls back to the id)."""
|
||||
try:
|
||||
body = self._call("/v1/user/detail", {"user_id": str(user_id)})
|
||||
except PixivAPIError:
|
||||
return None
|
||||
name = (body.get("user") or {}).get("name") if isinstance(body, dict) else None
|
||||
return name if isinstance(name, str) and name.strip() else None
|
||||
|
||||
# -- verify --------------------------------------------------------------
|
||||
|
||||
def verify_auth(self) -> tuple[bool | None, str]:
|
||||
"""Cheap credential probe: run the OAuth refresh (the thing that fails
|
||||
when the token is bad) without walking any feed."""
|
||||
try:
|
||||
self._token_deadline = 0.0 # force a real refresh
|
||||
self._login()
|
||||
except PixivAuthError as exc:
|
||||
return False, f"Pixiv rejected the credential — {exc}"
|
||||
except PixivAPIError as exc:
|
||||
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||
account = self._authed_user.get("account") or self._authed_user.get("name")
|
||||
suffix = f" as {account}" if account else ""
|
||||
return True, f"Credentials valid — Pixiv OAuth refresh succeeded{suffix}."
|
||||
|
||||
|
||||
def rating_label(x_restrict) -> str | None:
|
||||
"""Human rating from pixiv's x_restrict (0/1/2) — written into the post
|
||||
record so the archive keeps the R-18 flag without the reader needing to
|
||||
know pixiv's numeric scheme."""
|
||||
if isinstance(x_restrict, bool) or not isinstance(x_restrict, int):
|
||||
return None
|
||||
return _RATINGS.get(x_restrict)
|
||||
@@ -1,276 +0,0 @@
|
||||
"""Native Pixiv media downloader — the Pixiv counterpart to
|
||||
patreon_downloader / subscribestar_downloader.
|
||||
|
||||
Given a normalized Pixiv work and its resolved `MediaItem`s
|
||||
(pixiv_client.extract_media), download the originals to gallery-dl's on-disk
|
||||
layout (so pre-cutover gallery-dl downloads are recognized on disk and not
|
||||
re-fetched), write the post-first sidecars the importer consumes, and report
|
||||
per-media outcomes.
|
||||
|
||||
On-disk layout (matches FC's gallery-dl pixiv config, PLATFORM_DEFAULTS:
|
||||
base-directory `<images_root>/<artist_slug>/pixiv` + `directory:
|
||||
["{category}"]` + filename `{id}_{title[:50]}_{num:>02}.{extension}`):
|
||||
|
||||
<images_root>/<artist_slug>/pixiv/pixiv/<id>_<title50>_<NN>.<ext>
|
||||
|
||||
— note the intentional DOUBLE `pixiv` segment: gallery-dl appended
|
||||
`{category}` under a base-directory that already ended in the platform name,
|
||||
and tier-2 disk-skip parity requires reproducing that exactly. The layout is
|
||||
FLAT (no per-post directory), so the post-first record is `_post_<id>.json`
|
||||
in the same directory (the id suffix prevents the collisions a bare
|
||||
`_post.json` would have here; phase 3 receives explicit post_record_paths, so
|
||||
the name is a convention, not a discovery key).
|
||||
|
||||
Simpler than Patreon (no Mux/yt-dlp video branch) — the one special file is
|
||||
the ugoira frame zip, downloaded as-is; FC's archive-containment import
|
||||
extracts the frames, and the frame DELAYS ride the post record (the zip
|
||||
carries none — a future ugoira→video conversion needs them).
|
||||
|
||||
PURE: no DB; the seen-skip is an injected predicate. FC runs on a plain-HTTP
|
||||
homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
)
|
||||
from .pixiv_client import PIXIV_APP_HEADERS, rating_label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Control chars (0x00–0x1f + 0x7f DEL) — gallery-dl's default `path-remove`.
|
||||
_GDL_PATH_REMOVE_RE = re.compile(r"[\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
def gdl_clean_filename(name: str) -> str:
|
||||
"""Reproduce gallery-dl's on-disk filename EXACTLY as it wrote it on this
|
||||
Linux host, so the tier-2 disk-skip recognizes pre-cutover files instead of
|
||||
re-downloading them.
|
||||
|
||||
gallery-dl's PathFormat.build_filename is `clean_path(clean_segment(name))`.
|
||||
On Linux (verified against gallery-dl 1.32.5 path.py) the defaults resolve to:
|
||||
- path-restrict "auto" → "/" → clean_segment replaces ONLY "/" → "_"
|
||||
- path-remove "\\x00-\\x1f\\x7f" → clean_path DELETES control chars
|
||||
- path-strip "auto" → "" → NO trailing dot/space stripping
|
||||
Crucially it does NOT touch the Windows-forbidden set (<>:"|?*) — those stay
|
||||
raw in titles on disk. A stricter sanitizer here would rename any such title,
|
||||
miss the on-disk match, and re-pull the whole work. Order mirrors gallery-dl
|
||||
(segment inner, path outer); for these disjoint char sets it's commutative.
|
||||
"""
|
||||
return _GDL_PATH_REMOVE_RE.sub("", name.replace("/", "_"))
|
||||
|
||||
# Enrichment keys copied verbatim from the app-API work dict into the post
|
||||
# record (they're already JSON scalars/objects). Everything lands in
|
||||
# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and
|
||||
# structure without a schema change.
|
||||
_WORK_PASSTHROUGH_KEYS = (
|
||||
"type",
|
||||
"page_count",
|
||||
"width",
|
||||
"height",
|
||||
"total_view",
|
||||
"total_bookmarks",
|
||||
"total_comments",
|
||||
"is_bookmarked",
|
||||
"illust_ai_type",
|
||||
"series",
|
||||
)
|
||||
|
||||
|
||||
class PixivDownloader(BaseNativeDownloader):
|
||||
"""Download resolved Pixiv media to gallery-dl's on-disk layout.
|
||||
Subclasses BaseNativeDownloader for the shared streaming GET
|
||||
(transient-retry + Range-resume) and validation/quarantine. PURE: no DB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
ugoira_frames_fetcher: Callable[[dict], None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
images_root, cookies_path, platform="pixiv",
|
||||
validate=validate, rate_limit=rate_limit, session=session,
|
||||
)
|
||||
# Injected by the ingester (client.fetch_ugoira_frames) so write_post_record
|
||||
# can populate frame timings — which extract_media memoizes, but the core
|
||||
# writes the post record FIRST. Mirrors Patreon's content_fetcher.
|
||||
self._ugoira_frames_fetcher = ugoira_frames_fetcher
|
||||
if session is None:
|
||||
# i.pximg.net 403s any GET without the app Referer; mirror the
|
||||
# client's full app-header profile (gallery-dl serves media off
|
||||
# the same session it drives the API with). An injected session
|
||||
# (tests) owns its own headers.
|
||||
self.session.headers.update(PIXIV_APP_HEADERS)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one work; return per-item outcomes.
|
||||
Mirrors SubscribeStarDownloader.download_post (two-tier skip, mid-post
|
||||
time-box, recapture surfacing)."""
|
||||
flat_dir = self._flat_dir(artist_slug)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for media in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(
|
||||
post, media, flat_dir, artist_slug, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Pixiv media failed (work %s, %s): %s",
|
||||
post.get("id"), getattr(media, "media_id", "?"), exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def _flat_dir(self, artist_slug: str) -> Path:
|
||||
# Double platform segment — gallery-dl layout parity (module docstring).
|
||||
return self.images_root / artist_slug / "pixiv" / "pixiv"
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
flat_dir: Path,
|
||||
artist_slug: str,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
# The client's filename already carries the {id}_{title50}_{NN} shape
|
||||
# (raw title, gallery-dl-template order); clean it to the byte-exact
|
||||
# name gallery-dl wrote on disk so tier-2 disk-skip matches (else a
|
||||
# re-download of the whole work). See gdl_clean_filename.
|
||||
media_path = flat_dir / gdl_clean_filename(media.filename)
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
)
|
||||
# recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
flat_dir.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="quarantined", path=quarantine_dest, error=reason,
|
||||
)
|
||||
self._write_minimal_sidecar(post, out_path, source_url=media.url)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- post record ---------------------------------------------------------
|
||||
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
|
||||
"""Write the post-first `_post_<id>.json` — the sole writer of the post
|
||||
body/metadata on the native path. Beyond the standard body fields, the
|
||||
record carries pixiv's own structure (tags + EN translations, rating,
|
||||
series, view/bookmark counts, AI flag, dimensions, author, ugoira frame
|
||||
delays) so the archive keeps what the platform knows about the work."""
|
||||
attrs = post.get("attributes") or {}
|
||||
work = post.get("_work") or {}
|
||||
title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
|
||||
post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
return PostRecordOutcome(
|
||||
path=None, post_type=post_type, title=title, body_chars=0,
|
||||
)
|
||||
|
||||
content = attrs.get("content")
|
||||
content = content if isinstance(content, str) else ""
|
||||
data: dict = {
|
||||
"category": "pixiv",
|
||||
"id": pid,
|
||||
"title": title or "",
|
||||
"content": content,
|
||||
"published_at": attrs.get("published_at"),
|
||||
# The post permalink is synthesized by platforms/pixiv.py
|
||||
# derive_post_url from `id` at parse time — no url key here.
|
||||
"rating": rating_label(work.get("x_restrict")),
|
||||
}
|
||||
for key in _WORK_PASSTHROUGH_KEYS:
|
||||
if key in work:
|
||||
data[key] = work[key]
|
||||
tags = work.get("tags")
|
||||
if isinstance(tags, list):
|
||||
data["tags"] = [
|
||||
{
|
||||
"name": t.get("name"),
|
||||
"translated_name": t.get("translated_name"),
|
||||
}
|
||||
for t in tags
|
||||
if isinstance(t, dict)
|
||||
]
|
||||
user = work.get("user")
|
||||
if isinstance(user, dict):
|
||||
data["user"] = {
|
||||
"id": user.get("id"),
|
||||
"account": user.get("account"),
|
||||
"name": user.get("name"),
|
||||
}
|
||||
# Ugoira frame timings. extract_media memoizes these, but the core writes
|
||||
# the post record BEFORE extracting media, so fetch them here (shared +
|
||||
# idempotent via the client's memoization) so the record actually keeps
|
||||
# them — the zip carries no timings.
|
||||
if (
|
||||
work.get("type") == "ugoira"
|
||||
and not work.get("_ugoira_frames")
|
||||
and self._ugoira_frames_fetcher is not None
|
||||
):
|
||||
self._ugoira_frames_fetcher(post)
|
||||
frames = work.get("_ugoira_frames")
|
||||
if frames:
|
||||
data["ugoira_frames"] = frames
|
||||
|
||||
flat_dir = self._flat_dir(artist_slug)
|
||||
flat_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = flat_dir / f"_post_{pid}.json"
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=post_type, title=title, body_chars=len(content),
|
||||
)
|
||||
@@ -1,121 +0,0 @@
|
||||
"""Native Pixiv ingester — the Pixiv ADAPTER over the platform-agnostic core
|
||||
(`ingest_core.Ingester`).
|
||||
|
||||
Thin counterpart to patreon_ingester / subscribestar_ingester: wires the Pixiv
|
||||
client/downloader/ledger models/constraints/key into the core and supplies the
|
||||
Pixiv failure mapping. The modes (tick / backfill / recovery / recapture), the
|
||||
seen + dead-letter ledgers, cursor checkpointing, and the post-first capture
|
||||
all live in the core. `download_service.download_source` drives
|
||||
`PixivIngester.run` exactly as it drives the other two.
|
||||
|
||||
`campaign_id` is the numeric pixiv user id (download_backends extracts it from
|
||||
the source URL — no network resolver). Auth is the operator's OAuth refresh
|
||||
token (the token-type Credential), passed as `auth_token` — pixiv is the first
|
||||
native platform authenticating by token rather than cookies, so the uniform
|
||||
constructor accepts both and ignores what it doesn't need.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import PixivFailedMedia, PixivSeenMedia
|
||||
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
||||
from .pixiv_client import MediaItem, PixivAPIError, PixivClient
|
||||
from .pixiv_downloader import PixivDownloader
|
||||
|
||||
__all__ = [
|
||||
"DEAD_LETTER_THRESHOLD",
|
||||
"PixivIngester",
|
||||
"_ledger_key",
|
||||
"verify_pixiv_credential",
|
||||
]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""Stable per-media identity for the cross-run seen-ledger. Pixiv original
|
||||
URLs carry no content hash, so the key is the page/zip identity scoped to
|
||||
its work: `<illust_id>:p<num>` / `<illust_id>:ugoira`. Bounded to the
|
||||
column width."""
|
||||
if media.filehash:
|
||||
return media.filehash
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class PixivIngester(Ingester):
|
||||
"""Walk a pixiv user's works, download unseen originals, return a
|
||||
`DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
|
||||
`downloader` are injectable seams so unit tests run without network."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None,
|
||||
session_factory: Callable[[], object],
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: PixivClient | None = None,
|
||||
downloader: PixivDownloader | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
resolved_client = (
|
||||
client
|
||||
if client is not None
|
||||
else PixivClient(auth_token, request_sleep=request_sleep)
|
||||
)
|
||||
resolved_downloader = (
|
||||
downloader
|
||||
if downloader is not None
|
||||
else PixivDownloader(
|
||||
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
||||
# write_post_record runs before extract_media in the core, so it
|
||||
# fetches ugoira frame timings via the SAME client (shared,
|
||||
# memoized) — else the record's ugoira_frames stays empty.
|
||||
ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames,
|
||||
)
|
||||
)
|
||||
super().__init__(
|
||||
client=resolved_client,
|
||||
downloader=resolved_downloader,
|
||||
session_factory=session_factory,
|
||||
seen_model=PixivSeenMedia,
|
||||
failed_model=PixivFailedMedia,
|
||||
seen_constraint="uq_pixiv_seen_media_source_id",
|
||||
failed_constraint="uq_pixiv_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
platform="pixiv",
|
||||
error_base=PixivAPIError,
|
||||
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
||||
# the auth/drift/HTTP→error_type mapping (shared across platforms).
|
||||
drift_label="Pixiv app API",
|
||||
# Captions are legitimately empty for many pixiv artists, so the
|
||||
# zero-bodies #862 canary would false-positive here; the client's
|
||||
# response-shape checks (missing `illusts` → drift) cover the same
|
||||
# failure class structurally.
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
|
||||
async def verify_pixiv_credential(
|
||||
auth_token: str | None,
|
||||
) -> tuple[bool | None, str]:
|
||||
"""Native Pixiv credential probe — one OAuth refresh via
|
||||
PixivClient.verify_auth (the exact call that fails when the token is
|
||||
bad; no feed walk). Returns the uniform `(ok, message)` contract so
|
||||
download_backends.verify_source_credential treats it like the others."""
|
||||
client = PixivClient(auth_token)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, client.verify_auth)
|
||||
@@ -13,9 +13,8 @@ URL patterns match GS exactly so the existing browser extension
|
||||
hits FC unmodified. deviantart was dropped at #3069 (2026-08-27) —
|
||||
FC downloaders are art-dedicated services only. pixiv was retired at
|
||||
milestone #406 (2026-09-13, rule #171): unregistered here first, which
|
||||
switches it off everywhere this registry is consulted; `pixiv.py` and the
|
||||
pixiv client/downloader/ingester stay in the tree, uncalled, until the
|
||||
milestone's phase 2 deletes them.
|
||||
switched it off everywhere this registry is consulted, then removed from
|
||||
the tree entirely in the milestone's phase 2 (2026-09-21).
|
||||
"""
|
||||
|
||||
from .base import (
|
||||
|
||||
@@ -24,8 +24,8 @@ from typing import Literal
|
||||
# external_post_id chain: `post_id` MUST come before `id` because
|
||||
# SubscribeStar gallery-dl puts the per-attachment id in `id` and the
|
||||
# actual post id in `post_id`; picking `id` first fragments
|
||||
# multi-image SubscribeStar posts into N Post rows. Patreon/Pixiv have
|
||||
# no `post_id` so `id` still wins for them; HF uses `index`, Discord
|
||||
# multi-image SubscribeStar posts into N Post rows. Patreon has
|
||||
# no `post_id` so `id` still wins for it; HF uses `index`, Discord
|
||||
# uses `message_id` — all reached via the remaining chain entries.
|
||||
# (Banked 2026-05-27 during the sidecar audit.)
|
||||
DEFAULT_EXTERNAL_POST_ID_KEYS: tuple[str, ...] = (
|
||||
@@ -62,7 +62,7 @@ class PlatformInfo:
|
||||
# --- Behavioral hooks ---
|
||||
# Synthesize a post permalink from sidecar data. Required when
|
||||
# gallery-dl's `url` field is the file/CDN URL rather than the post
|
||||
# permalink (subscribestar/pixiv/hf/discord). None = trust the bare
|
||||
# permalink (subscribestar/hf/discord). None = trust the bare
|
||||
# `url` field (patreon).
|
||||
derive_post_url: Callable[[dict], str | None] | None = None
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Pixiv — one quirk.
|
||||
|
||||
post_url: the sidecar's `url` (legacy gallery-dl era) is the image URL
|
||||
on `i.pximg.net`, and the native post record (#129) writes no url key
|
||||
at all — the permalink is synthesized from `id` here either way:
|
||||
/artworks/<id>. external_post_id (= `id`) was already correct, so no
|
||||
override there.
|
||||
|
||||
Downloads run through the native ingester (pixiv_ingester.py), not
|
||||
gallery-dl; this registry entry still owns URL validation, sidecar
|
||||
parsing, and the credential surface (the OAuth refresh token).
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
|
||||
|
||||
def derive_post_url(data: dict) -> str | None:
|
||||
pid = str_id_value(data.get("id"))
|
||||
if pid:
|
||||
return f"https://www.pixiv.net/artworks/{pid}"
|
||||
return None
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
key="pixiv",
|
||||
name="Pixiv",
|
||||
description="Download artwork from Pixiv artists",
|
||||
auth_type="token",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?pixiv\.net/",
|
||||
url_examples=[
|
||||
"https://www.pixiv.net/users/12345678",
|
||||
"https://www.pixiv.net/en/users/12345678",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["all"]},
|
||||
notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.",
|
||||
derive_post_url=derive_post_url,
|
||||
)
|
||||
+1
-2
@@ -1,7 +1,7 @@
|
||||
# FabledCurator Firefox Extension
|
||||
|
||||
Self-hosted Firefox extension that pushes session cookies from supported
|
||||
platforms (Patreon, SubscribeStar, Hentai-Foundry, Discord, Pixiv)
|
||||
platforms (Patreon, SubscribeStar, Hentai-Foundry, Discord)
|
||||
into FabledCurator, and lets you add a creator as a Source from their
|
||||
page in one click.
|
||||
|
||||
@@ -33,7 +33,6 @@ npm run build # unsigned XPI in web-ext-artifacts/
|
||||
- [ ] Options page accepts FC URL + key, indicator turns green
|
||||
- [ ] Cookie export: log into patreon.com, click Patreon card → "X cookies exported"
|
||||
- [ ] Discord token: open discord.com, click Discord card → "Token captured"
|
||||
- [ ] Pixiv OAuth: click Pixiv card → login redirects, token stored
|
||||
- [ ] Add as source: visit patreon.com/<creator>, click floating button → toast
|
||||
- [ ] Subscriptions list: popup → "Sources" tab → list renders
|
||||
- [ ] Check now: click play icon on source row → no error toast
|
||||
|
||||
-129
@@ -1,129 +0,0 @@
|
||||
{
|
||||
"illusts": [
|
||||
{
|
||||
"id": 111,
|
||||
"title": "Multi Page Adventure",
|
||||
"type": "illust",
|
||||
"caption": "Two-page set. <a href=\"https://example.com/wip\">WIP thread</a>",
|
||||
"create_date": "2026-06-20T18:00:00+09:00",
|
||||
"user": {"id": 99, "name": "Example Artist", "account": "exartist"},
|
||||
"tags": [
|
||||
{"name": "オリジナル", "translated_name": "original"},
|
||||
{"name": "女の子", "translated_name": "girl"}
|
||||
],
|
||||
"page_count": 2,
|
||||
"width": 1200,
|
||||
"height": 1600,
|
||||
"x_restrict": 0,
|
||||
"series": {"id": 4242, "title": "Adventure Series"},
|
||||
"total_view": 1000,
|
||||
"total_bookmarks": 250,
|
||||
"is_bookmarked": false,
|
||||
"illust_ai_type": 1,
|
||||
"meta_single_page": {},
|
||||
"meta_pages": [
|
||||
{
|
||||
"image_urls": {
|
||||
"square_medium": "https://i.pximg.net/c/360x360_70/img-master/img/2026/06/20/18/00/00/111_p0_square1200.jpg",
|
||||
"original": "https://i.pximg.net/img-original/img/2026/06/20/18/00/00/111_p0.png"
|
||||
}
|
||||
},
|
||||
{
|
||||
"image_urls": {
|
||||
"square_medium": "https://i.pximg.net/c/360x360_70/img-master/img/2026/06/20/18/00/00/111_p1_square1200.jpg",
|
||||
"original": "https://i.pximg.net/img-original/img/2026/06/20/18/00/00/111_p1.png"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 222,
|
||||
"title": "Single Piece",
|
||||
"type": "illust",
|
||||
"caption": "",
|
||||
"create_date": "2026-06-18T12:30:00+09:00",
|
||||
"user": {"id": 99, "name": "Example Artist", "account": "exartist"},
|
||||
"tags": [{"name": "落書き", "translated_name": "doodle"}],
|
||||
"page_count": 1,
|
||||
"width": 900,
|
||||
"height": 900,
|
||||
"x_restrict": 1,
|
||||
"series": null,
|
||||
"total_view": 500,
|
||||
"total_bookmarks": 60,
|
||||
"is_bookmarked": true,
|
||||
"illust_ai_type": 0,
|
||||
"meta_single_page": {
|
||||
"original_image_url": "https://i.pximg.net/img-original/img/2026/06/18/12/30/00/222_p0.jpg"
|
||||
},
|
||||
"meta_pages": []
|
||||
},
|
||||
{
|
||||
"id": 333,
|
||||
"title": "Wiggle Loop",
|
||||
"type": "ugoira",
|
||||
"caption": "animated",
|
||||
"create_date": "2026-06-15T09:00:00+09:00",
|
||||
"user": {"id": 99, "name": "Example Artist", "account": "exartist"},
|
||||
"tags": [{"name": "うごイラ", "translated_name": "ugoira"}],
|
||||
"page_count": 1,
|
||||
"width": 600,
|
||||
"height": 600,
|
||||
"x_restrict": 0,
|
||||
"series": null,
|
||||
"total_view": 300,
|
||||
"total_bookmarks": 40,
|
||||
"is_bookmarked": false,
|
||||
"illust_ai_type": 0,
|
||||
"meta_single_page": {
|
||||
"original_image_url": "https://i.pximg.net/img-original/img/2026/06/15/09/00/00/333_ugoira0.jpg"
|
||||
},
|
||||
"meta_pages": []
|
||||
},
|
||||
{
|
||||
"id": 444,
|
||||
"title": "Blocked Work",
|
||||
"type": "illust",
|
||||
"caption": "",
|
||||
"create_date": "2026-06-10T00:00:00+09:00",
|
||||
"user": {"id": 99, "name": "Example Artist", "account": "exartist"},
|
||||
"tags": [],
|
||||
"page_count": 1,
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"x_restrict": 2,
|
||||
"series": null,
|
||||
"total_view": 0,
|
||||
"total_bookmarks": 0,
|
||||
"is_bookmarked": false,
|
||||
"illust_ai_type": 0,
|
||||
"meta_single_page": {
|
||||
"original_image_url": "https://s.pximg.net/common/images/limit_sanity_level_360.png"
|
||||
},
|
||||
"meta_pages": []
|
||||
},
|
||||
{
|
||||
"id": 555,
|
||||
"title": "Ghost Work",
|
||||
"type": "illust",
|
||||
"caption": "",
|
||||
"create_date": "2026-06-01T00:00:00+09:00",
|
||||
"user": {"id": 0, "name": "", "account": ""},
|
||||
"tags": [],
|
||||
"page_count": 1,
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"x_restrict": 0,
|
||||
"series": null,
|
||||
"total_view": 0,
|
||||
"total_bookmarks": 0,
|
||||
"is_bookmarked": false,
|
||||
"illust_ai_type": 0,
|
||||
"meta_single_page": {
|
||||
"original_image_url": "https://i.pximg.net/img-original/img/2026/06/01/00/00/00/555_p0.png"
|
||||
},
|
||||
"meta_pages": []
|
||||
}
|
||||
],
|
||||
"next_url": "https://app-api.pixiv.net/v1/user/illusts?user_id=99&offset=30"
|
||||
}
|
||||
@@ -87,29 +87,21 @@ async def test_quick_add_reuses_source_artist_after_rename(client, ext_key):
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch):
|
||||
# #130: each native platform resolves its real display name at add-time
|
||||
# (pixiv=token API, patreon=campaigns API, subscribestar=profile page);
|
||||
# gallery-dl platforms and any failure fall back to the URL handle.
|
||||
# (patreon=campaigns API, subscribestar=profile page); gallery-dl platforms
|
||||
# and any failure fall back to the URL handle.
|
||||
from backend.app.services import patreon_resolver
|
||||
from backend.app.services.credential_service import CredentialService
|
||||
from backend.app.services.extension_service import ExtensionService
|
||||
from backend.app.services.pixiv_client import PixivClient
|
||||
from backend.app.services.subscribestar_client import SubscribeStarClient
|
||||
|
||||
async def _tok(self, platform):
|
||||
return "tok"
|
||||
|
||||
async def _cookies(self, platform):
|
||||
return "/tmp/cookies.txt"
|
||||
|
||||
monkeypatch.setattr(CredentialService, "get_token", _tok)
|
||||
monkeypatch.setattr(CredentialService, "get_cookies_path", _cookies)
|
||||
monkeypatch.setattr(PixivClient, "resolve_display_name", lambda self, uid: "Pixiv Name")
|
||||
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: "Patreon Name")
|
||||
monkeypatch.setattr(SubscribeStarClient, "resolve_display_name", lambda self, u: "SS Name")
|
||||
|
||||
svc = ExtensionService(db, crypto=object()) # crypto seam only (calls stubbed)
|
||||
assert await svc._resolve_artist_name(
|
||||
"pixiv", "555", "https://www.pixiv.net/users/555") == "Pixiv Name"
|
||||
assert await svc._resolve_artist_name(
|
||||
"patreon", "maewix", "https://patreon.com/maewix") == "Patreon Name"
|
||||
assert await svc._resolve_artist_name(
|
||||
@@ -117,7 +109,7 @@ async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch):
|
||||
# gallery-dl platform → readable handle passthrough (no resolver).
|
||||
assert await svc._resolve_artist_name("hentaifoundry", "Foo", "u") == "Foo"
|
||||
# No crypto → no resolution attempt → the raw handle.
|
||||
assert await ExtensionService(db)._resolve_artist_name("pixiv", "555", "u") == "555"
|
||||
assert await ExtensionService(db)._resolve_artist_name("patreon", "maewix", "u") == "maewix"
|
||||
# Resolver returns None → fall back to the handle.
|
||||
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: None)
|
||||
assert await svc._resolve_artist_name("patreon", "maewix", "u") == "maewix"
|
||||
|
||||
@@ -7,7 +7,6 @@ import pytest
|
||||
|
||||
from backend.app.services.download_backends import (
|
||||
NATIVE_INGESTER_PLATFORMS,
|
||||
_campaign_resolution_error,
|
||||
_native_ingester_cls,
|
||||
_unsupported_platform_message,
|
||||
run_download,
|
||||
@@ -15,7 +14,6 @@ from backend.app.services.download_backends import (
|
||||
verify_source_credential,
|
||||
)
|
||||
from backend.app.services.gallery_dl import ErrorType
|
||||
from backend.app.services.pixiv_ingester import PixivIngester
|
||||
|
||||
|
||||
def test_native_platforms():
|
||||
@@ -25,8 +23,9 @@ def test_native_platforms():
|
||||
|
||||
|
||||
def test_pixiv_is_no_longer_native():
|
||||
"""Retired at milestone #406. The refusal below is what stops it falling
|
||||
through to gallery-dl now that it is not native."""
|
||||
"""Retired at milestone #406 — unregistered in phase 1, deleted in phase 2.
|
||||
The refusal below is what stops it falling through to gallery-dl now that
|
||||
it is neither native nor registered."""
|
||||
assert uses_native_ingester("pixiv") is False
|
||||
assert "pixiv" not in NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
@@ -113,11 +112,14 @@ def test_unknown_platform_is_not_native():
|
||||
assert uses_native_ingester("nonsense") is False
|
||||
|
||||
|
||||
def test_pixiv_dispatches_to_its_ingester():
|
||||
assert _native_ingester_cls("pixiv") is PixivIngester
|
||||
def test_every_native_platform_dispatches_to_an_ingester():
|
||||
"""The dispatch table and NATIVE_INGESTER_PLATFORMS have to agree, or a
|
||||
platform that routes native raises KeyError mid-download instead of being
|
||||
refused up front. Written over the set rather than per-platform so adding
|
||||
one to NATIVE_INGESTER_PLATFORMS and forgetting the class fails here.
|
||||
|
||||
(This replaces the per-platform dispatch tests, one of which was pixiv's;
|
||||
it was deleted with pixiv at milestone #406 phase 2.)"""
|
||||
for platform in NATIVE_INGESTER_PLATFORMS:
|
||||
assert _native_ingester_cls(platform) is not None
|
||||
|
||||
def test_pixiv_resolution_error_names_the_expected_url_shape():
|
||||
msg = _campaign_resolution_error("pixiv", "https://www.pixiv.net/artworks/1")
|
||||
assert "pixiv user id" in msg
|
||||
assert "users/<id>" in msg
|
||||
|
||||
@@ -1,419 +0,0 @@
|
||||
"""PixivClient tests — parsing + iteration against canned pages, no network.
|
||||
|
||||
The fixture mirrors a real `/v1/user/illusts` page (multi-page work, single
|
||||
page, ugoira, sanity-limited placeholder, deleted-author ghost). HTTP is
|
||||
stubbed at the requests-session seam (oauth POST + API GET), so the exact
|
||||
gallery-dl-parity request profile — headers, oauth form, pagination params —
|
||||
is asserted rather than assumed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.pixiv_client import (
|
||||
PIXIV_APP_HEADERS,
|
||||
MediaItem,
|
||||
PixivAPIError,
|
||||
PixivAuthError,
|
||||
PixivClient,
|
||||
PixivDriftError,
|
||||
rating_label,
|
||||
user_id_from_url,
|
||||
)
|
||||
|
||||
_FIXTURE = Path(__file__).parent / "fixtures" / "pixiv_user_illusts_page1.json"
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, status_code=200, json_data=None, headers=None):
|
||||
self.status_code = status_code
|
||||
self._json = json_data
|
||||
self.headers = headers or {}
|
||||
|
||||
def json(self):
|
||||
if self._json is None:
|
||||
raise ValueError("no JSON")
|
||||
return self._json
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""Minimal requests.Session stand-in: canned responses per (method, url
|
||||
fragment), recording every call for profile assertions."""
|
||||
|
||||
def __init__(self):
|
||||
self.headers = dict(PIXIV_APP_HEADERS)
|
||||
self.responses = []
|
||||
self.calls = []
|
||||
|
||||
def queue(self, response):
|
||||
self.responses.append(response)
|
||||
return self
|
||||
|
||||
def _next(self):
|
||||
if not self.responses:
|
||||
raise AssertionError("FakeSession ran out of queued responses")
|
||||
return self.responses.pop(0)
|
||||
|
||||
def post(self, url, data=None, headers=None, timeout=None):
|
||||
self.calls.append(("POST", url, data, headers))
|
||||
return self._next()
|
||||
|
||||
def get(self, url, params=None, timeout=None, headers=None):
|
||||
self.calls.append(("GET", url, params, headers))
|
||||
return self._next()
|
||||
|
||||
|
||||
def _oauth_ok():
|
||||
return FakeResponse(200, {
|
||||
"response": {
|
||||
"access_token": "acc-token",
|
||||
"expires_in": 3600,
|
||||
"user": {"id": "77", "account": "operator", "name": "Op"},
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def page1():
|
||||
return json.loads(_FIXTURE.read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
# Never issues a request in pure-parsing tests.
|
||||
return PixivClient("refresh-tok", session=FakeSession())
|
||||
|
||||
|
||||
def _post_for(client, page1, work_id):
|
||||
for work in page1["illusts"]:
|
||||
if work["id"] == work_id:
|
||||
return client._normalize(work)
|
||||
raise AssertionError(f"no work {work_id} in fixture")
|
||||
|
||||
|
||||
# -- URL → user id ----------------------------------------------------------
|
||||
|
||||
def test_user_id_from_url_variants():
|
||||
assert user_id_from_url("https://www.pixiv.net/users/12345678") == "12345678"
|
||||
assert user_id_from_url("https://www.pixiv.net/en/users/42") == "42"
|
||||
assert user_id_from_url("https://pixiv.net/users/7/artworks") == "7"
|
||||
assert user_id_from_url("https://www.pixiv.net/member.php?id=99") == "99"
|
||||
|
||||
|
||||
def test_user_id_from_url_rejects_non_matches():
|
||||
assert user_id_from_url("https://www.pixiv.net/artworks/111") is None
|
||||
assert user_id_from_url("https://www.pixiv.net/users/notdigits") is None
|
||||
assert user_id_from_url("https://example.com/users/5") is None
|
||||
assert user_id_from_url("") is None
|
||||
|
||||
|
||||
# -- normalization ------------------------------------------------------------
|
||||
|
||||
def test_normalize_maps_attributes(client, page1):
|
||||
post = _post_for(client, page1, 111)
|
||||
attrs = post["attributes"]
|
||||
assert post["id"] == 111
|
||||
assert attrs["title"] == "Multi Page Adventure"
|
||||
assert "Two-page set." in attrs["content"]
|
||||
assert attrs["published_at"] == "2026-06-20T18:00:00+09:00"
|
||||
assert attrs["post_type"] == "illust"
|
||||
assert post["_work"]["total_bookmarks"] == 250
|
||||
|
||||
|
||||
def test_post_record_key_and_meta(client, page1):
|
||||
post = _post_for(client, page1, 222)
|
||||
assert client.post_record_key(post) == ("post:222", "222")
|
||||
meta = client.post_meta(post)
|
||||
assert meta["title"] == "Single Piece"
|
||||
assert meta["date"] == "2026-06-18T12:30:00+09:00"
|
||||
assert client.post_record_key({"id": None}) is None
|
||||
|
||||
|
||||
# -- gating -------------------------------------------------------------------
|
||||
|
||||
def test_post_is_gated_limit_placeholder(client, page1):
|
||||
assert client.post_is_gated(_post_for(client, page1, 444)) is True
|
||||
|
||||
|
||||
def test_post_is_gated_deleted_author(client, page1):
|
||||
assert client.post_is_gated(_post_for(client, page1, 555)) is True
|
||||
|
||||
|
||||
def test_post_is_gated_normal_works(client, page1):
|
||||
assert client.post_is_gated(_post_for(client, page1, 111)) is False
|
||||
assert client.post_is_gated(_post_for(client, page1, 222)) is False
|
||||
|
||||
|
||||
# -- extract_media --------------------------------------------------------------
|
||||
|
||||
def test_extract_media_multi_page(client, page1):
|
||||
items = client.extract_media(_post_for(client, page1, 111), {})
|
||||
assert len(items) == 2
|
||||
assert all(isinstance(m, MediaItem) for m in items)
|
||||
assert [m.media_id for m in items] == ["p0", "p1"]
|
||||
assert items[0].url.endswith("/111_p0.png")
|
||||
assert items[1].url.endswith("/111_p1.png")
|
||||
# gallery-dl filename parity: {id}_{title[:50]}_{num:>02}.{extension}
|
||||
assert items[0].filename == "111_Multi Page Adventure_00.png"
|
||||
assert items[1].filename == "111_Multi Page Adventure_01.png"
|
||||
assert all(m.post_id == "111" for m in items)
|
||||
|
||||
|
||||
def test_extract_media_single_page(client, page1):
|
||||
items = client.extract_media(_post_for(client, page1, 222), {})
|
||||
assert len(items) == 1
|
||||
assert items[0].media_id == "p0"
|
||||
assert items[0].url.endswith("/222_p0.jpg")
|
||||
assert items[0].filename == "222_Single Piece_00.jpg"
|
||||
|
||||
|
||||
def test_extract_media_gated_yields_nothing(client, page1):
|
||||
assert client.extract_media(_post_for(client, page1, 444), {}) == []
|
||||
assert client.extract_media(_post_for(client, page1, 555), {}) == []
|
||||
|
||||
|
||||
def test_extract_media_ugoira_zip_swap(client, page1, monkeypatch):
|
||||
frames = [{"file": "000000.jpg", "delay": 90}, {"file": "000001.jpg", "delay": 90}]
|
||||
|
||||
def fake_call(endpoint, params):
|
||||
assert endpoint == "/v1/ugoira/metadata"
|
||||
assert params == {"illust_id": "333"}
|
||||
return {"ugoira_metadata": {
|
||||
"zip_urls": {"medium": (
|
||||
"https://i.pximg.net/img-zip-ugoira/img/2026/06/15/09/00/00/"
|
||||
"333_ugoira600x600.zip"
|
||||
)},
|
||||
"frames": frames,
|
||||
}}
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
post = _post_for(client, page1, 333)
|
||||
items = client.extract_media(post, {})
|
||||
assert len(items) == 1
|
||||
assert items[0].media_id == "ugoira"
|
||||
assert items[0].kind == "ugoira"
|
||||
assert items[0].url.endswith("333_ugoira1920x1080.zip")
|
||||
assert items[0].filename == "333_Wiggle Loop_00.zip"
|
||||
# Frame delays memoized for the post record (future video conversion).
|
||||
assert post["_work"]["_ugoira_frames"] == frames
|
||||
|
||||
|
||||
def test_fetch_ugoira_frames_memoizes_and_shares_one_call(client, page1, monkeypatch):
|
||||
# fetch_ugoira_frames (called by write_post_record, which the core runs
|
||||
# BEFORE extract_media) populates frames; extract_media then REUSES the
|
||||
# memoized metadata — exactly one /v1/ugoira/metadata call total.
|
||||
frames = [{"file": "000000.jpg", "delay": 90}]
|
||||
calls = []
|
||||
|
||||
def fake_call(endpoint, params):
|
||||
calls.append(endpoint)
|
||||
return {"ugoira_metadata": {
|
||||
"zip_urls": {"medium": (
|
||||
"https://i.pximg.net/img-zip-ugoira/x/333_ugoira600x600.zip"
|
||||
)},
|
||||
"frames": frames,
|
||||
}}
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
post = _post_for(client, page1, 333)
|
||||
client.fetch_ugoira_frames(post)
|
||||
assert post["_work"]["_ugoira_frames"] == frames
|
||||
items = client.extract_media(post, {}) # reuses memoized meta
|
||||
assert items[0].media_id == "ugoira"
|
||||
assert calls == ["/v1/ugoira/metadata"] # ONE fetch, not two
|
||||
|
||||
|
||||
def test_fetch_ugoira_frames_noop_for_non_ugoira(client, page1, monkeypatch):
|
||||
monkeypatch.setattr(client, "_call", lambda e, p: pytest.fail("should not fetch"))
|
||||
post = _post_for(client, page1, 222) # a single-page illust
|
||||
client.fetch_ugoira_frames(post)
|
||||
assert "_ugoira_frames" not in post["_work"]
|
||||
|
||||
|
||||
def test_extract_media_ugoira_metadata_failure_downgrades(
|
||||
client, page1, monkeypatch
|
||||
):
|
||||
def fake_call(endpoint, params):
|
||||
raise PixivAPIError("boom", status_code=500)
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
assert client.extract_media(_post_for(client, page1, 333), {}) == []
|
||||
|
||||
|
||||
def test_extract_media_ugoira_auth_failure_stays_loud(
|
||||
client, page1, monkeypatch
|
||||
):
|
||||
def fake_call(endpoint, params):
|
||||
raise PixivAuthError("token dead", status_code=400)
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
with pytest.raises(PixivAuthError):
|
||||
client.extract_media(_post_for(client, page1, 333), {})
|
||||
|
||||
|
||||
# -- iteration ------------------------------------------------------------------
|
||||
|
||||
def test_iter_posts_paginates_and_carries_cursor(page1, monkeypatch):
|
||||
client = PixivClient("refresh-tok", session=FakeSession())
|
||||
page2 = {
|
||||
"illusts": [dict(page1["illusts"][1], id=666, title="Older")],
|
||||
"next_url": None,
|
||||
}
|
||||
seen_params = []
|
||||
|
||||
def fake_call(endpoint, params):
|
||||
assert endpoint == "/v1/user/illusts"
|
||||
seen_params.append(dict(params))
|
||||
return page1 if len(seen_params) == 1 else page2
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
rows = list(client.iter_posts("99"))
|
||||
|
||||
assert seen_params[0] == {"user_id": "99"}
|
||||
# Page 2 params come verbatim from next_url's query string.
|
||||
assert seen_params[1] == {"user_id": "99", "offset": "30"}
|
||||
# Page-1 posts carry cursor None; page-2 posts carry the fetching cursor.
|
||||
assert [c for _, _, c in rows[:5]] == [None] * 5
|
||||
assert rows[5][2] == "user_id=99&offset=30"
|
||||
assert rows[5][0]["id"] == 666
|
||||
|
||||
|
||||
def test_iter_posts_resumes_from_cursor(page1, monkeypatch):
|
||||
client = PixivClient("refresh-tok", session=FakeSession())
|
||||
captured = {}
|
||||
|
||||
def fake_call(endpoint, params):
|
||||
captured["params"] = dict(params)
|
||||
return {"illusts": [], "next_url": None}
|
||||
|
||||
monkeypatch.setattr(client, "_call", fake_call)
|
||||
list(client.iter_posts("99", cursor="user_id=99&offset=60"))
|
||||
assert captured["params"] == {"user_id": "99", "offset": "60"}
|
||||
|
||||
|
||||
def test_iter_posts_rejects_non_numeric_campaign():
|
||||
client = PixivClient("refresh-tok", session=FakeSession())
|
||||
with pytest.raises(PixivDriftError):
|
||||
next(client.iter_posts("https://www.pixiv.net/users/99"))
|
||||
|
||||
|
||||
def test_iter_posts_drift_on_missing_illusts(monkeypatch):
|
||||
client = PixivClient("refresh-tok", session=FakeSession())
|
||||
monkeypatch.setattr(client, "_call", lambda e, p: {"error": None, "body": 1})
|
||||
with pytest.raises(PixivDriftError):
|
||||
next(client.iter_posts("99"))
|
||||
|
||||
|
||||
# -- auth ------------------------------------------------------------------------
|
||||
|
||||
def test_login_sends_gallery_dl_profile_and_sets_bearer():
|
||||
session = FakeSession().queue(_oauth_ok())
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
client._login()
|
||||
|
||||
method, url, data, headers = session.calls[0]
|
||||
assert method == "POST"
|
||||
assert url == "https://oauth.secure.pixiv.net/auth/token"
|
||||
assert data["grant_type"] == "refresh_token"
|
||||
assert data["refresh_token"] == "refresh-tok"
|
||||
assert data["client_id"] == "MOBrBDS8blbauoSck0ZfDbtuzpyT"
|
||||
assert data["get_secure_url"] == "1"
|
||||
# X-Client-Time/-Hash pair: ISO + literal +00:00, md5 hex digest.
|
||||
assert headers["X-Client-Time"].endswith("+00:00")
|
||||
assert len(headers["X-Client-Hash"]) == 32
|
||||
assert session.headers["Authorization"] == "Bearer acc-token"
|
||||
# Token is fresh → a second login is a no-op (no extra POST).
|
||||
client._login()
|
||||
assert len(session.calls) == 1
|
||||
|
||||
|
||||
def test_login_maps_rejection_to_auth_error():
|
||||
session = FakeSession().queue(FakeResponse(400, {"has_error": True}))
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
with pytest.raises(PixivAuthError):
|
||||
client._login()
|
||||
|
||||
|
||||
def test_login_without_token_is_auth_error():
|
||||
client = PixivClient(None, session=FakeSession())
|
||||
with pytest.raises(PixivAuthError):
|
||||
client._login()
|
||||
|
||||
|
||||
def test_call_maps_rate_limit_message():
|
||||
session = FakeSession().queue(_oauth_ok()).queue(FakeResponse(403, {
|
||||
"error": {"message": "Rate Limit", "user_message": ""},
|
||||
}))
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
with pytest.raises(PixivAPIError) as exc_info:
|
||||
client._call("/v1/user/illusts", {"user_id": "1"})
|
||||
err = exc_info.value
|
||||
assert not isinstance(err, PixivAuthError)
|
||||
assert err.status_code == 429
|
||||
assert err.retry_after == 300.0
|
||||
|
||||
|
||||
def test_call_maps_403_to_auth_error():
|
||||
session = FakeSession().queue(_oauth_ok()).queue(FakeResponse(403, {
|
||||
"error": {"message": "invalid access token", "user_message": ""},
|
||||
}))
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
with pytest.raises(PixivAuthError):
|
||||
client._call("/v1/user/illusts", {"user_id": "1"})
|
||||
|
||||
|
||||
def test_call_maps_404_status():
|
||||
session = FakeSession().queue(_oauth_ok()).queue(FakeResponse(404, {
|
||||
"error": {"message": "Not Found", "user_message": ""},
|
||||
}))
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
with pytest.raises(PixivAPIError) as exc_info:
|
||||
client._call("/v1/user/illusts", {"user_id": "1"})
|
||||
assert exc_info.value.status_code == 404
|
||||
|
||||
|
||||
def test_verify_auth_reports_account():
|
||||
session = FakeSession().queue(_oauth_ok())
|
||||
client = PixivClient("refresh-tok", session=session)
|
||||
ok, message = client.verify_auth()
|
||||
assert ok is True
|
||||
assert "operator" in message
|
||||
|
||||
|
||||
def test_verify_auth_bad_token():
|
||||
session = FakeSession().queue(FakeResponse(400, {"has_error": True}))
|
||||
client = PixivClient("bad", session=session)
|
||||
ok, message = client.verify_auth()
|
||||
assert ok is False
|
||||
assert "rotate" in message.lower() or "rejected" in message.lower()
|
||||
|
||||
|
||||
def test_resolve_display_name(client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
client, "_call",
|
||||
lambda e, p: {"user": {"name": "Kurotsuchi Machi", "id": p["user_id"]}},
|
||||
)
|
||||
assert client.resolve_display_name("99") == "Kurotsuchi Machi"
|
||||
|
||||
|
||||
def test_resolve_display_name_none_on_failure(client, monkeypatch):
|
||||
def boom(endpoint, params):
|
||||
raise PixivAPIError("nope", status_code=404)
|
||||
monkeypatch.setattr(client, "_call", boom)
|
||||
assert client.resolve_display_name("99") is None
|
||||
# Empty/whitespace name → None (caller falls back to the id).
|
||||
monkeypatch.setattr(client, "_call", lambda e, p: {"user": {"name": " "}})
|
||||
assert client.resolve_display_name("99") is None
|
||||
|
||||
|
||||
# -- rating ------------------------------------------------------------------------
|
||||
|
||||
def test_rating_label():
|
||||
assert rating_label(0) == "General"
|
||||
assert rating_label(1) == "R-18"
|
||||
assert rating_label(2) == "R-18G"
|
||||
assert rating_label(None) is None
|
||||
assert rating_label(True) is None
|
||||
assert rating_label(9) is None
|
||||
@@ -1,309 +0,0 @@
|
||||
"""Unit tests for PixivDownloader — no network, no DB.
|
||||
|
||||
The HTTP layer is stubbed via the `session` seam (a fake session whose `.get`
|
||||
returns canned streaming bytes). Media items come from the real
|
||||
PixivClient.extract_media over the shared fixture page, so the downloader is
|
||||
exercised against the exact shapes the client produces.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from backend.app.services.pixiv_client import MediaItem, PixivClient
|
||||
from backend.app.services.pixiv_downloader import PixivDownloader
|
||||
|
||||
_FIXTURE = Path(__file__).parent / "fixtures" / "pixiv_user_illusts_page1.json"
|
||||
|
||||
# Minimal valid PNG so the file_validator passes on the happy path.
|
||||
_PNG_HEAD = b"\x89PNG\r\n\x1a\n"
|
||||
_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: bytes, status_code: int = 200, headers=None):
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise requests.HTTPError(f"HTTP {self.status_code}")
|
||||
return None
|
||||
|
||||
def iter_content(self, chunk_size=65536):
|
||||
for i in range(0, len(self._payload), chunk_size):
|
||||
yield self._payload[i : i + chunk_size]
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Records GETs and returns canned bytes (per-URL, default _PNG_BYTES)."""
|
||||
|
||||
def __init__(self, payloads: dict[str, bytes] | None = None):
|
||||
self.payloads = payloads or {}
|
||||
self.calls: list[str] = []
|
||||
|
||||
def get(self, url, stream=False, timeout=None, headers=None):
|
||||
self.calls.append(url)
|
||||
return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
|
||||
|
||||
|
||||
def _client():
|
||||
# Parsing only — extract_media for non-ugoira works never issues a request.
|
||||
return PixivClient("tok", session=_FakeSession())
|
||||
|
||||
|
||||
def _post_and_media(work_id):
|
||||
page = json.loads(_FIXTURE.read_text())
|
||||
client = _client()
|
||||
for work in page["illusts"]:
|
||||
if work["id"] == work_id:
|
||||
post = client._normalize(work)
|
||||
return post, client.extract_media(post, {})
|
||||
raise AssertionError(f"no work {work_id} in fixture")
|
||||
|
||||
|
||||
def _post_only(work_id):
|
||||
"""Normalized post WITHOUT media extraction — the write_post_record tests
|
||||
don't need media, and extracting an ugoira would hit the API seam."""
|
||||
page = json.loads(_FIXTURE.read_text())
|
||||
for work in page["illusts"]:
|
||||
if work["id"] == work_id:
|
||||
return _client()._normalize(work)
|
||||
raise AssertionError(f"no work {work_id} in fixture")
|
||||
|
||||
|
||||
def _downloader(tmp_path, session=None):
|
||||
# validate=False: the stub payload is PNG bytes regardless of the target
|
||||
# extension, so real validation would quarantine the .jpg cases. The
|
||||
# validate/quarantine plumbing is BaseNativeDownloader's, covered by the
|
||||
# Patreon downloader tests.
|
||||
return PixivDownloader(
|
||||
tmp_path, validate=False,
|
||||
session=session if session is not None else _FakeSession(),
|
||||
)
|
||||
|
||||
|
||||
def test_download_post_writes_gallery_dl_layout(tmp_path):
|
||||
post, media = _post_and_media(111)
|
||||
dl = _downloader(tmp_path)
|
||||
outcomes = dl.download_post(post, media, "artist-a")
|
||||
|
||||
assert [o.status for o in outcomes] == ["downloaded", "downloaded"]
|
||||
flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
|
||||
p0 = flat / "111_Multi Page Adventure_00.png"
|
||||
p1 = flat / "111_Multi Page Adventure_01.png"
|
||||
assert p0.is_file()
|
||||
assert p1.is_file()
|
||||
# No per-post directory — pixiv's layout is flat (gallery-dl parity).
|
||||
assert {p.parent for p in (p0, p1)} == {flat}
|
||||
|
||||
|
||||
def test_download_post_writes_minimal_sidecar(tmp_path):
|
||||
post, media = _post_and_media(222)
|
||||
dl = _downloader(tmp_path)
|
||||
outcomes = dl.download_post(post, media, "artist-a")
|
||||
|
||||
assert outcomes[0].status == "downloaded"
|
||||
sidecar = outcomes[0].path.with_suffix(".json")
|
||||
data = json.loads(sidecar.read_text())
|
||||
# Post-first: image identity ONLY — the body lives in the post record.
|
||||
assert data == {
|
||||
"category": "pixiv",
|
||||
"id": "222",
|
||||
"source_url": media[0].url,
|
||||
}
|
||||
|
||||
|
||||
def test_download_post_skips_seen(tmp_path):
|
||||
post, media = _post_and_media(222)
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(post, media, "artist-a", is_seen=lambda m: True)
|
||||
assert [o.status for o in outcomes] == ["skipped_seen"]
|
||||
assert session.calls == []
|
||||
|
||||
|
||||
def test_download_post_skips_on_disk(tmp_path):
|
||||
post, media = _post_and_media(222)
|
||||
flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
|
||||
flat.mkdir(parents=True)
|
||||
existing = flat / "222_Single Piece_00.jpg"
|
||||
existing.write_bytes(b"already here")
|
||||
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(post, media, "artist-a")
|
||||
assert [o.status for o in outcomes] == ["skipped_disk"]
|
||||
assert outcomes[0].path == existing
|
||||
assert session.calls == []
|
||||
assert existing.read_bytes() == b"already here"
|
||||
|
||||
|
||||
def test_recapture_surfaces_on_disk_path_for_seen_media(tmp_path):
|
||||
post, media = _post_and_media(222)
|
||||
flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
|
||||
flat.mkdir(parents=True)
|
||||
(flat / "222_Single Piece_00.jpg").write_bytes(b"kept")
|
||||
|
||||
dl = _downloader(tmp_path)
|
||||
outcomes = dl.download_post(
|
||||
post, media, "artist-a", is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
# On disk → surfaced with its path (relink channel); seen-but-missing
|
||||
# stays skipped_seen (recovery's job, not recapture's).
|
||||
assert outcomes[0].status == "skipped_disk"
|
||||
assert outcomes[0].path is not None
|
||||
|
||||
|
||||
def test_recapture_does_not_redownload_missing_seen_media(tmp_path):
|
||||
post, media = _post_and_media(222)
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(
|
||||
post, media, "artist-a", is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
assert [o.status for o in outcomes] == ["skipped_seen"]
|
||||
assert session.calls == []
|
||||
|
||||
|
||||
def test_download_post_honours_should_stop(tmp_path):
|
||||
post, media = _post_and_media(111)
|
||||
dl = _downloader(tmp_path)
|
||||
outcomes = dl.download_post(post, media, "artist-a", should_stop=lambda: True)
|
||||
assert outcomes == []
|
||||
|
||||
|
||||
def test_filename_matches_gallery_dl_linux_restrict(tmp_path):
|
||||
# gallery-dl on Linux replaces ONLY "/" and deletes control chars — the
|
||||
# Windows-forbidden set (<>:"|?*) stays RAW in the on-disk name. Matching
|
||||
# that byte-for-byte is what lets the tier-2 disk-skip recognize
|
||||
# pre-cutover files; a stricter sanitizer would re-download them.
|
||||
post, _ = _post_and_media(222)
|
||||
weird = MediaItem(
|
||||
url="https://i.pximg.net/img-original/img/2026/06/18/12/30/00/222_p0.jpg",
|
||||
filename='222_What? A "Title"/Slash\x0900.jpg', # ? " stay, / → _, \t dropped
|
||||
kind="image",
|
||||
filehash=None,
|
||||
post_id="222",
|
||||
media_id="p0",
|
||||
)
|
||||
dl = _downloader(tmp_path)
|
||||
outcomes = dl.download_post(post, [weird], "artist-a")
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert outcomes[0].path.name == '222_What? A "Title"_Slash00.jpg'
|
||||
|
||||
|
||||
def test_gdl_clean_filename_parity():
|
||||
from backend.app.services.pixiv_downloader import gdl_clean_filename
|
||||
# Only "/" is replaced; the Windows-forbidden set is untouched.
|
||||
assert gdl_clean_filename('a<>:"|?*b.png') == 'a<>:"|?*b.png'
|
||||
assert gdl_clean_filename("a/b/c.png") == "a_b_c.png"
|
||||
# Control chars (newline, tab, DEL) are deleted, not replaced.
|
||||
assert gdl_clean_filename("a\nb\tc\x7fd.png") == "abcd.png"
|
||||
# Trailing dots/spaces are NOT stripped on Linux.
|
||||
assert gdl_clean_filename("title . .png") == "title . .png"
|
||||
|
||||
|
||||
def test_own_session_carries_app_referer(tmp_path):
|
||||
# Built session (no injection) must carry the app profile — i.pximg.net
|
||||
# 403s GETs without the app-api Referer.
|
||||
dl = PixivDownloader(tmp_path)
|
||||
assert dl.session.headers["Referer"] == "https://app-api.pixiv.net/"
|
||||
assert dl.session.headers["App-OS"] == "ios"
|
||||
|
||||
|
||||
def test_write_post_record_enriched(tmp_path):
|
||||
post = _post_only(111)
|
||||
dl = _downloader(tmp_path)
|
||||
outcome = dl.write_post_record(post, "artist-a")
|
||||
|
||||
assert outcome.path == (
|
||||
tmp_path / "artist-a" / "pixiv" / "pixiv" / "_post_111.json"
|
||||
)
|
||||
assert outcome.title == "Multi Page Adventure"
|
||||
assert outcome.post_type == "illust"
|
||||
assert outcome.body_chars == len(post["attributes"]["content"])
|
||||
|
||||
data = json.loads(outcome.path.read_text())
|
||||
assert data["category"] == "pixiv"
|
||||
assert data["id"] == "111"
|
||||
assert data["published_at"] == "2026-06-20T18:00:00+09:00"
|
||||
assert data["rating"] == "General"
|
||||
assert data["page_count"] == 2
|
||||
assert data["total_bookmarks"] == 250
|
||||
assert data["total_view"] == 1000
|
||||
assert data["illust_ai_type"] == 1
|
||||
assert data["series"] == {"id": 4242, "title": "Adventure Series"}
|
||||
assert data["tags"] == [
|
||||
{"name": "オリジナル", "translated_name": "original"},
|
||||
{"name": "女の子", "translated_name": "girl"},
|
||||
]
|
||||
assert data["user"] == {"id": 99, "account": "exartist", "name": "Example Artist"}
|
||||
# JP tags stay human-readable on disk (ensure_ascii=False).
|
||||
assert "オリジナル" in outcome.path.read_text()
|
||||
|
||||
|
||||
def test_write_post_record_rating_r18(tmp_path):
|
||||
post = _post_only(222)
|
||||
dl = _downloader(tmp_path)
|
||||
outcome = dl.write_post_record(post, "artist-a")
|
||||
data = json.loads(outcome.path.read_text())
|
||||
assert data["rating"] == "R-18"
|
||||
assert data["is_bookmarked"] is True
|
||||
|
||||
|
||||
def test_write_post_record_includes_ugoira_frames(tmp_path):
|
||||
post = _post_only(333)
|
||||
frames = [{"file": "000000.jpg", "delay": 90}]
|
||||
post["_work"]["_ugoira_frames"] = frames
|
||||
dl = _downloader(tmp_path)
|
||||
outcome = dl.write_post_record(post, "artist-a")
|
||||
data = json.loads(outcome.path.read_text())
|
||||
assert data["type"] == "ugoira"
|
||||
assert data["ugoira_frames"] == frames
|
||||
|
||||
|
||||
def test_write_post_record_fetches_ugoira_frames_when_absent(tmp_path):
|
||||
# The core writes the post record BEFORE extract_media, so frames aren't
|
||||
# memoized yet — the injected fetcher must populate them so the record keeps
|
||||
# the timings (regression: they were silently always empty).
|
||||
post = _post_only(333)
|
||||
assert "_ugoira_frames" not in post["_work"]
|
||||
frames = [{"file": "000000.jpg", "delay": 120}]
|
||||
calls = []
|
||||
|
||||
def fetcher(p):
|
||||
calls.append(p)
|
||||
p["_work"]["_ugoira_frames"] = frames
|
||||
|
||||
dl = PixivDownloader(
|
||||
tmp_path, validate=False, session=_FakeSession(),
|
||||
ugoira_frames_fetcher=fetcher,
|
||||
)
|
||||
outcome = dl.write_post_record(post, "artist-a")
|
||||
data = json.loads(outcome.path.read_text())
|
||||
assert data["ugoira_frames"] == frames
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_write_post_record_non_ugoira_does_not_call_fetcher(tmp_path):
|
||||
post = _post_only(111) # a plain multi-page illust
|
||||
calls = []
|
||||
dl = PixivDownloader(
|
||||
tmp_path, validate=False, session=_FakeSession(),
|
||||
ugoira_frames_fetcher=lambda p: calls.append(p),
|
||||
)
|
||||
dl.write_post_record(post, "artist-a")
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_write_post_record_without_id(tmp_path):
|
||||
dl = _downloader(tmp_path)
|
||||
outcome = dl.write_post_record({"id": None, "attributes": {}}, "artist-a")
|
||||
assert outcome.path is None
|
||||
assert outcome.body_chars == 0
|
||||
@@ -1,305 +0,0 @@
|
||||
"""PixivIngester tests — the adapter's wiring over the shared core.
|
||||
|
||||
The core's walk/skip/cursor/budget behavior is exercised exhaustively by
|
||||
test_patreon_ingester / test_subscribestar_native; these cover what is
|
||||
pixiv-SPECIFIC: the synthesized ledger key, the real Postgres pixiv ledgers
|
||||
(the upserts' constraint names must match migration 0076), the failure
|
||||
mapping including the rate-limit retry_after carry, and the #862 body-canary
|
||||
opt-out (caption-less pixiv feeds must not fail API_DRIFT).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from backend.app.models import Artist, PixivFailedMedia, PixivSeenMedia, Source
|
||||
from backend.app.services.gallery_dl import ErrorType
|
||||
from backend.app.services.ingest_core import _CANARY_MIN_SAMPLE
|
||||
from backend.app.services.native_ingest_common import MediaOutcome, PostRecordOutcome
|
||||
from backend.app.services.pixiv_client import (
|
||||
MediaItem,
|
||||
PixivAPIError,
|
||||
PixivAuthError,
|
||||
PixivDriftError,
|
||||
)
|
||||
from backend.app.services.pixiv_ingester import PixivIngester, _ledger_key
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
# --- fakes ----------------------------------------------------------------
|
||||
|
||||
|
||||
def _media(post_id, num, *, filehash=None, media_id=None):
|
||||
mid = media_id if media_id is not None else f"p{num}"
|
||||
return MediaItem(
|
||||
url=f"https://i.pximg.net/img-original/img/2026/07/01/00/00/00/{post_id}_{mid}.png",
|
||||
filename=f"{post_id}_Work_{num:02d}.png",
|
||||
kind="image",
|
||||
filehash=filehash,
|
||||
post_id=str(post_id),
|
||||
media_id=mid,
|
||||
)
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
"""Stub PixivClient. `pages` is a list of (page_cursor, [posts]); each post
|
||||
is (post_id, [MediaItem]). `raise_exc` trips a client-level failure;
|
||||
`captions` toggles per-post body text (the canary input)."""
|
||||
|
||||
def __init__(self, pages, raise_exc=None, captions=True):
|
||||
self._pages = pages
|
||||
self._raise_exc = raise_exc
|
||||
self._captions = captions
|
||||
|
||||
def iter_posts(self, campaign_id, cursor=None):
|
||||
if self._raise_exc is not None:
|
||||
raise self._raise_exc
|
||||
for page_cursor, posts in self._pages:
|
||||
for post_id, media in posts:
|
||||
caption = f"<p>caption {post_id}</p>" if self._captions else ""
|
||||
yield (
|
||||
{
|
||||
"id": post_id,
|
||||
"_media": media,
|
||||
"attributes": {
|
||||
"title": f"Work {post_id}",
|
||||
"post_type": "illust",
|
||||
"content": caption,
|
||||
"published_at": "2026-07-01T00:00:00+09:00",
|
||||
},
|
||||
"_work": {},
|
||||
},
|
||||
{},
|
||||
page_cursor,
|
||||
)
|
||||
|
||||
def extract_media(self, post, included_index):
|
||||
return post["_media"]
|
||||
|
||||
def post_meta(self, post):
|
||||
return {"title": post.get("id"), "date": None}
|
||||
|
||||
@staticmethod
|
||||
def post_is_gated(post):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def post_record_key(post):
|
||||
pid = str(post.get("id") or "")
|
||||
return (f"post:{pid}", pid) if pid else None
|
||||
|
||||
|
||||
class _FakeDownloader:
|
||||
"""Stub PixivDownloader. Honors the injected is_seen; media whose ledger
|
||||
key is in `error` report a download failure."""
|
||||
|
||||
def __init__(self, tmp_path, error=None):
|
||||
self.tmp_path = tmp_path
|
||||
self.error = set(error or ())
|
||||
self.download_calls = 0
|
||||
|
||||
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
||||
should_stop=lambda: False, recapture=False):
|
||||
outcomes = []
|
||||
for m in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
if is_seen(m) and not recapture:
|
||||
outcomes.append(
|
||||
MediaOutcome(media=m, status="skipped_seen", path=None, error=None)
|
||||
)
|
||||
elif _ledger_key(m) in self.error:
|
||||
self.download_calls += 1
|
||||
outcomes.append(
|
||||
MediaOutcome(media=m, status="error", path=None, error="403 pximg")
|
||||
)
|
||||
else:
|
||||
self.download_calls += 1
|
||||
p = self.tmp_path / m.filename
|
||||
p.write_bytes(b"x")
|
||||
outcomes.append(
|
||||
MediaOutcome(media=m, status="downloaded", path=p, error=None)
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def write_post_record(self, post, artist_slug):
|
||||
p = self.tmp_path / f"_post_{post.get('id')}.json"
|
||||
p.write_text("{}")
|
||||
attrs = post.get("attributes") or {}
|
||||
body = attrs.get("content")
|
||||
return PostRecordOutcome(
|
||||
path=p,
|
||||
post_type=attrs.get("post_type"),
|
||||
title=attrs.get("title"),
|
||||
body_chars=len(body) if isinstance(body, str) else 0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def source_id(db):
|
||||
artist = Artist(name="Pixland", slug="pixland")
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
source = Source(
|
||||
artist_id=artist.id, platform="pixiv",
|
||||
url="https://www.pixiv.net/users/99", enabled=True, config_overrides={},
|
||||
)
|
||||
db.add(source)
|
||||
await db.commit()
|
||||
return source.id
|
||||
|
||||
|
||||
def _ingester(sync_engine, tmp_path, client, downloader):
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
return PixivIngester(
|
||||
images_root=tmp_path, cookies_path=None, session_factory=factory,
|
||||
client=client, downloader=downloader,
|
||||
)
|
||||
|
||||
|
||||
def _run(ing, source_id, mode="tick"):
|
||||
return ing.run(
|
||||
source_id=source_id, campaign_id="99", artist_slug="pixland",
|
||||
url="https://www.pixiv.net/users/99", mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def _seen_keys(sync_engine, source_id):
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
return set(s.execute(
|
||||
select(PixivSeenMedia.filehash).where(
|
||||
PixivSeenMedia.source_id == source_id
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
|
||||
# --- ledger key -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ledger_key_shapes():
|
||||
assert _ledger_key(_media(111, 0)) == "111:p0"
|
||||
assert _ledger_key(_media(111, 4)) == "111:p4"
|
||||
assert _ledger_key(_media(333, 0, media_id="ugoira")) == "333:ugoira"
|
||||
assert _ledger_key(_media(1, 0, filehash="f" * 32)) == "f" * 32
|
||||
long_key = _ledger_key(_media("9" * 200, 0))
|
||||
assert len(long_key) <= 128
|
||||
|
||||
|
||||
# --- walk + ledgers ---------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_downloads_and_marks_seen(source_id, sync_engine, tmp_path):
|
||||
m0, m1 = _media(111, 0), _media(111, 1)
|
||||
client = _FakeClient([(None, [(111, [m0, m1])])])
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = _run(ing, source_id)
|
||||
|
||||
assert result.success is True
|
||||
assert result.files_downloaded == 2
|
||||
assert len(result.post_record_paths) == 1
|
||||
# 2 media keys + the synthetic post key, in the pixiv-shaped key format.
|
||||
assert _seen_keys(sync_engine, source_id) == {"111:p0", "111:p1", "post:111"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
|
||||
m0 = _media(111, 0)
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
s.add(PixivSeenMedia(
|
||||
source_id=source_id, filehash=_ledger_key(m0), post_id="111",
|
||||
))
|
||||
s.commit()
|
||||
|
||||
client = _FakeClient([(None, [(111, [m0])])])
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = _run(ing, source_id)
|
||||
assert result.files_downloaded == 0
|
||||
assert downloader.download_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_media_lands_in_dead_letter_ledger(
|
||||
source_id, sync_engine, tmp_path
|
||||
):
|
||||
m0 = _media(111, 0)
|
||||
client = _FakeClient([(None, [(111, [m0])])])
|
||||
downloader = _FakeDownloader(tmp_path, error={_ledger_key(m0)})
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = _run(ing, source_id)
|
||||
assert result.run_stats["per_item_failures"] == 1
|
||||
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
row = s.execute(
|
||||
select(PixivFailedMedia).where(
|
||||
PixivFailedMedia.source_id == source_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert row.filehash == "111:p0"
|
||||
assert row.attempts == 1
|
||||
assert "403" in row.last_error
|
||||
|
||||
|
||||
# --- failure mapping ----------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
|
||||
client = _FakeClient([], raise_exc=PixivAuthError(
|
||||
"rotate the token", status_code=400,
|
||||
))
|
||||
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
||||
result = _run(ing, source_id)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.AUTH_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drift_error_maps_to_api_drift(source_id, sync_engine, tmp_path):
|
||||
client = _FakeClient([], raise_exc=PixivDriftError("no illusts list"))
|
||||
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
||||
result = _run(ing, source_id)
|
||||
assert result.error_type == ErrorType.API_DRIFT
|
||||
assert "Pixiv app API changed" in result.error_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limit_carries_retry_after(source_id, sync_engine, tmp_path):
|
||||
client = _FakeClient([], raise_exc=PixivAPIError(
|
||||
"rate limited", status_code=429, retry_after=300.0,
|
||||
))
|
||||
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
||||
result = _run(ing, source_id)
|
||||
assert result.error_type == ErrorType.RATE_LIMITED
|
||||
assert result.retry_after_seconds == 300.0
|
||||
|
||||
|
||||
# --- body canary opt-out --------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_captionless_feed_does_not_trip_body_canary(
|
||||
source_id, sync_engine, tmp_path
|
||||
):
|
||||
"""#862 canary opt-out: a pixiv backfill recording ≥ the canary sample of
|
||||
caption-less works is NORMAL (many artists never write captions) — it must
|
||||
complete, not fail API_DRIFT the way a zero-body Patreon walk does."""
|
||||
n = _CANARY_MIN_SAMPLE + 5
|
||||
posts = [(1000 + i, [_media(1000 + i, 0)]) for i in range(n)]
|
||||
client = _FakeClient([(None, posts)], captions=False)
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = _run(ing, source_id, mode="backfill")
|
||||
assert result.success is True
|
||||
assert result.error_type is None
|
||||
assert result.files_downloaded == n
|
||||
@@ -26,6 +26,40 @@ def test_pixiv_is_retired():
|
||||
assert "pixiv" not in known_platform_keys()
|
||||
|
||||
|
||||
def test_pixiv_code_and_tables_are_gone():
|
||||
"""Milestone #406 phase 2 (step 7). Phase 1 made pixiv unreachable; this is
|
||||
the guard that it stays deleted rather than drifting back in.
|
||||
|
||||
Both halves assert absence from a DATA STRUCTURE — the module table and the
|
||||
declarative metadata — not from prose. Snippet #3352's trap is an absence
|
||||
check against a comment, which passes the moment someone rewords the
|
||||
comment; a re-added module or a re-declared table cannot hide from these.
|
||||
|
||||
DeviantArt is the reason this exists: #3069 retired it in code on
|
||||
2026-08-27 and its credential row was still sitting in the database seven
|
||||
weeks later (#3980). A retirement nothing asserts is a retirement that
|
||||
half-happens.
|
||||
"""
|
||||
import importlib
|
||||
|
||||
from backend.app.models.base import Base
|
||||
|
||||
for module in (
|
||||
"backend.app.services.pixiv_client",
|
||||
"backend.app.services.pixiv_downloader",
|
||||
"backend.app.services.pixiv_ingester",
|
||||
"backend.app.services.platforms.pixiv",
|
||||
"backend.app.models.pixiv_seen_media",
|
||||
"backend.app.models.pixiv_failed_media",
|
||||
):
|
||||
with pytest.raises(ModuleNotFoundError):
|
||||
importlib.import_module(module)
|
||||
|
||||
# Alembic 0102 drops these; nothing may re-declare them.
|
||||
assert "pixiv_seen_media" not in Base.metadata.tables
|
||||
assert "pixiv_failed_media" not in Base.metadata.tables
|
||||
|
||||
|
||||
def test_fanbox_not_in_registry():
|
||||
# Sanity check — FC-3a added 'fanbox' by mistake; it's not a GS platform.
|
||||
assert "fanbox" not in PLATFORMS
|
||||
|
||||
Reference in New Issue
Block a user