bvandeusen
f33808b977
The core writes the post record BEFORE extract_media, but the ugoira frame delays were only memoized DURING extract_media — so write_post_record never saw them and ugoira_frames was always empty in the record. Extract a memoized _ugoira_meta (frames + zip url share ONE /v1/ugoira/metadata call regardless of order) and inject client.fetch_ugoira_frames into the downloader (mirrors Patreon's content_fetcher) so write_post_record populates the frames itself. Zero extra API calls — the fetch is shared/memoized with extract_media. A recapture now backfills the timings onto existing ugoira posts. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
122 lines
4.8 KiB
Python
122 lines
4.8 KiB
Python
"""Native Pixiv ingester — the Pixiv ADAPTER over the platform-agnostic core
|
|
(`ingest_core.Ingester`).
|
|
|
|
Thin counterpart to patreon_ingester / subscribestar_ingester: wires the Pixiv
|
|
client/downloader/ledger models/constraints/key into the core and supplies the
|
|
Pixiv failure mapping. The modes (tick / backfill / recovery / recapture), the
|
|
seen + dead-letter ledgers, cursor checkpointing, and the post-first capture
|
|
all live in the core. `download_service.download_source` drives
|
|
`PixivIngester.run` exactly as it drives the other two.
|
|
|
|
`campaign_id` is the numeric pixiv user id (download_backends extracts it from
|
|
the source URL — no network resolver). Auth is the operator's OAuth refresh
|
|
token (the token-type Credential), passed as `auth_token` — pixiv is the first
|
|
native platform authenticating by token rather than cookies, so the uniform
|
|
constructor accepts both and ignores what it doesn't need.
|
|
|
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
|
|
from ..models import PixivFailedMedia, PixivSeenMedia
|
|
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
|
from .pixiv_client import MediaItem, PixivAPIError, PixivClient
|
|
from .pixiv_downloader import PixivDownloader
|
|
|
|
__all__ = [
|
|
"DEAD_LETTER_THRESHOLD",
|
|
"PixivIngester",
|
|
"_ledger_key",
|
|
"verify_pixiv_credential",
|
|
]
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_LEDGER_KEY_MAX = 128
|
|
|
|
|
|
def _ledger_key(media: MediaItem) -> str:
|
|
"""Stable per-media identity for the cross-run seen-ledger. Pixiv original
|
|
URLs carry no content hash, so the key is the page/zip identity scoped to
|
|
its work: `<illust_id>:p<num>` / `<illust_id>:ugoira`. Bounded to the
|
|
column width."""
|
|
if media.filehash:
|
|
return media.filehash
|
|
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
|
|
|
|
|
class PixivIngester(Ingester):
|
|
"""Walk a pixiv user's works, download unseen originals, return a
|
|
`DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
|
|
`downloader` are injectable seams so unit tests run without network."""
|
|
|
|
def __init__(
|
|
self,
|
|
images_root: Path,
|
|
cookies_path: str | None,
|
|
session_factory: Callable[[], object],
|
|
*,
|
|
validate: bool = True,
|
|
rate_limit: float = 0.0,
|
|
request_sleep: float = 0.0,
|
|
auth_token: str | None = None,
|
|
client: PixivClient | None = None,
|
|
downloader: PixivDownloader | None = None,
|
|
):
|
|
self.images_root = Path(images_root)
|
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
|
resolved_client = (
|
|
client
|
|
if client is not None
|
|
else PixivClient(auth_token, request_sleep=request_sleep)
|
|
)
|
|
resolved_downloader = (
|
|
downloader
|
|
if downloader is not None
|
|
else PixivDownloader(
|
|
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
|
# write_post_record runs before extract_media in the core, so it
|
|
# fetches ugoira frame timings via the SAME client (shared,
|
|
# memoized) — else the record's ugoira_frames stays empty.
|
|
ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames,
|
|
)
|
|
)
|
|
super().__init__(
|
|
client=resolved_client,
|
|
downloader=resolved_downloader,
|
|
session_factory=session_factory,
|
|
seen_model=PixivSeenMedia,
|
|
failed_model=PixivFailedMedia,
|
|
seen_constraint="uq_pixiv_seen_media_source_id",
|
|
failed_constraint="uq_pixiv_failed_media_source_id",
|
|
ledger_key=_ledger_key,
|
|
platform="pixiv",
|
|
error_base=PixivAPIError,
|
|
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
|
# the auth/drift/HTTP→error_type mapping (shared across platforms).
|
|
drift_label="Pixiv app API",
|
|
# Captions are legitimately empty for many pixiv artists, so the
|
|
# zero-bodies #862 canary would false-positive here; the client's
|
|
# response-shape checks (missing `illusts` → drift) cover the same
|
|
# failure class structurally.
|
|
body_canary=False,
|
|
)
|
|
|
|
|
|
async def verify_pixiv_credential(
|
|
auth_token: str | None,
|
|
) -> tuple[bool | None, str]:
|
|
"""Native Pixiv credential probe — one OAuth refresh via
|
|
PixivClient.verify_auth (the exact call that fails when the token is
|
|
bad; no feed walk). Returns the uniform `(ok, message)` contract so
|
|
download_backends.verify_source_credential treats it like the others."""
|
|
client = PixivClient(auth_token)
|
|
loop = asyncio.get_running_loop()
|
|
return await loop.run_in_executor(None, client.verify_auth)
|