From 7ef2ecd82f08b03577c0b535dc16a34b6e9cfdc9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Jul 2026 09:40:59 -0400 Subject: [PATCH] =?UTF-8?q?feat(pixiv):=20native=20downloader=20=E2=80=94?= =?UTF-8?q?=20gallery-dl=20layout=20parity=20+=20enriched=20post=20record?= =?UTF-8?q?=20(#129=20step=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PixivDownloader writes originals to the exact pre-cutover gallery-dl layout (/pixiv/pixiv/{id}_{title[:50]}_{NN}.{ext} — flat, double platform segment) so tier-2 disk-skip recognizes existing files. Post-first: per-media sidecar is identity-only; the post record (_post_.json — id suffix because the flat layout would collide a bare _post.json) carries the enrichment: tags + EN translations, rating from x_restrict, series, view/bookmark/comment counts, AI flag, dimensions, author, and ugoira frame delays (the zip has no timings). i.pximg.net media GETs ride the app-header profile (403 without the app-api Referer). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/services/pixiv_downloader.py | 239 ++++++++++++++++++++++ tests/test_pixiv_downloader.py | 245 +++++++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 backend/app/services/pixiv_downloader.py create mode 100644 tests/test_pixiv_downloader.py diff --git a/backend/app/services/pixiv_downloader.py b/backend/app/services/pixiv_downloader.py new file mode 100644 index 0000000..63bca1e --- /dev/null +++ b/backend/app/services/pixiv_downloader.py @@ -0,0 +1,239 @@ +"""Native Pixiv media downloader — the Pixiv counterpart to +patreon_downloader / subscribestar_downloader. + +Given a normalized Pixiv work and its resolved `MediaItem`s +(pixiv_client.extract_media), download the originals to gallery-dl's on-disk +layout (so pre-cutover gallery-dl downloads are recognized on disk and not +re-fetched), write the post-first sidecars the importer consumes, and report +per-media outcomes. + +On-disk layout (matches FC's gallery-dl pixiv config, PLATFORM_DEFAULTS: +base-directory `//pixiv` + `directory: +["{category}"]` + filename `{id}_{title[:50]}_{num:>02}.{extension}`): + + //pixiv/pixiv/__. + +— note the intentional DOUBLE `pixiv` segment: gallery-dl appended +`{category}` under a base-directory that already ended in the platform name, +and tier-2 disk-skip parity requires reproducing that exactly. The layout is +FLAT (no per-post directory), so the post-first record is `_post_.json` +in the same directory (the id suffix prevents the collisions a bare +`_post.json` would have here; phase 3 receives explicit post_record_paths, so +the name is a convention, not a discovery key). + +Simpler than Patreon (no Mux/yt-dlp video branch) — the one special file is +the ugoira frame zip, downloaded as-is; FC's archive-containment import +extracts the frames, and the frame DELAYS ride the post record (the zip +carries none — a future ugoira→video conversion needs them). + +PURE: no DB; the seen-skip is an injected predicate. FC runs on a plain-HTTP +homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import json +import logging +import time +from collections.abc import Callable +from pathlib import Path + +import requests + +from .native_ingest_common import ( + BaseNativeDownloader, + MediaOutcome, + PostRecordOutcome, + sanitize_segment, +) +from .pixiv_client import PIXIV_APP_HEADERS, rating_label + +log = logging.getLogger(__name__) + +# Enrichment keys copied verbatim from the app-API work dict into the post +# record (they're already JSON scalars/objects). Everything lands in +# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and +# structure without a schema change. +_WORK_PASSTHROUGH_KEYS = ( + "type", + "page_count", + "width", + "height", + "total_view", + "total_bookmarks", + "total_comments", + "is_bookmarked", + "illust_ai_type", + "series", +) + + +class PixivDownloader(BaseNativeDownloader): + """Download resolved Pixiv media to gallery-dl's on-disk layout. + Subclasses BaseNativeDownloader for the shared streaming GET + (transient-retry + Range-resume) and validation/quarantine. PURE: no DB.""" + + def __init__( + self, + images_root: Path, + cookies_path: str | None = None, + *, + validate: bool = True, + rate_limit: float = 0.0, + session: requests.Session | None = None, + ): + super().__init__( + images_root, cookies_path, platform="pixiv", + validate=validate, rate_limit=rate_limit, session=session, + ) + if session is None: + # i.pximg.net 403s any GET without the app Referer; mirror the + # client's full app-header profile (gallery-dl serves media off + # the same session it drives the API with). An injected session + # (tests) owns its own headers. + self.session.headers.update(PIXIV_APP_HEADERS) + + # -- public ------------------------------------------------------------ + + def download_post( + self, + post: dict, + media_items: list, + artist_slug: str, + *, + is_seen: Callable[[object], bool] = lambda m: False, + should_stop: Callable[[], bool] = lambda: False, + recapture: bool = False, + ) -> list[MediaOutcome]: + """Download every media item of one work; return per-item outcomes. + Mirrors SubscribeStarDownloader.download_post (two-tier skip, mid-post + time-box, recapture surfacing).""" + flat_dir = self._flat_dir(artist_slug) + outcomes: list[MediaOutcome] = [] + for media in media_items: + if should_stop(): + break + try: + outcomes.append( + self._download_one( + post, media, flat_dir, artist_slug, is_seen, + recapture=recapture, + ) + ) + except Exception as exc: # resilient: isolate one item's failure + log.warning( + "Pixiv media failed (work %s, %s): %s", + post.get("id"), getattr(media, "media_id", "?"), exc, + ) + outcomes.append( + MediaOutcome(media=media, status="error", path=None, error=str(exc)) + ) + return outcomes + + def _flat_dir(self, artist_slug: str) -> Path: + # Double platform segment — gallery-dl layout parity (module docstring). + return self.images_root / artist_slug / "pixiv" / "pixiv" + + # -- per-item ---------------------------------------------------------- + + def _download_one( + self, + post: dict, + media, + flat_dir: Path, + artist_slug: str, + is_seen: Callable[[object], bool], + *, + recapture: bool = False, + ) -> MediaOutcome: + seen = is_seen(media) + if seen and not recapture: + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + + # The client's filename already carries the {id}_{title50}_{NN} shape; + # only path-restrict sanitization happens here. + media_path = flat_dir / sanitize_segment(media.filename) + + if media_path.exists(): # tier-2: already on disk + return MediaOutcome( + media=media, status="skipped_disk", path=media_path, error=None + ) + # recapture: a seen item not on disk is NOT re-downloaded (recovery's job). + if seen: + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + + flat_dir.mkdir(parents=True, exist_ok=True) + if self._rate_limit > 0: + time.sleep(self._rate_limit) + + out_path = self._fetch_get(media.url, media_path) + reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url) + if reason is not None: + return MediaOutcome( + media=media, status="quarantined", path=quarantine_dest, error=reason, + ) + self._write_minimal_sidecar(post, out_path, source_url=media.url) + return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) + + # -- post record --------------------------------------------------------- + + def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome: + """Write the post-first `_post_.json` — the sole writer of the post + body/metadata on the native path. Beyond the standard body fields, the + record carries pixiv's own structure (tags + EN translations, rating, + series, view/bookmark counts, AI flag, dimensions, author, ugoira frame + delays) so the archive keeps what the platform knows about the work.""" + attrs = post.get("attributes") or {} + work = post.get("_work") or {} + title = attrs.get("title") if isinstance(attrs.get("title"), str) else None + post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None + pid = str(post.get("id") or "") + if not pid: + return PostRecordOutcome( + path=None, post_type=post_type, title=title, body_chars=0, + ) + + content = attrs.get("content") + content = content if isinstance(content, str) else "" + data: dict = { + "category": "pixiv", + "id": pid, + "title": title or "", + "content": content, + "published_at": attrs.get("published_at"), + # The post permalink is synthesized by platforms/pixiv.py + # derive_post_url from `id` at parse time — no url key here. + "rating": rating_label(work.get("x_restrict")), + } + for key in _WORK_PASSTHROUGH_KEYS: + if key in work: + data[key] = work[key] + tags = work.get("tags") + if isinstance(tags, list): + data["tags"] = [ + { + "name": t.get("name"), + "translated_name": t.get("translated_name"), + } + for t in tags + if isinstance(t, dict) + ] + user = work.get("user") + if isinstance(user, dict): + data["user"] = { + "id": user.get("id"), + "account": user.get("account"), + "name": user.get("name"), + } + # Memoized by extract_media's ugoira branch; the zip has no timings. + frames = work.get("_ugoira_frames") + if frames: + data["ugoira_frames"] = frames + + flat_dir = self._flat_dir(artist_slug) + flat_dir.mkdir(parents=True, exist_ok=True) + path = flat_dir / f"_post_{pid}.json" + path.write_text(json.dumps(data, indent=2, ensure_ascii=False)) + return PostRecordOutcome( + path=path, post_type=post_type, title=title, body_chars=len(content), + ) diff --git a/tests/test_pixiv_downloader.py b/tests/test_pixiv_downloader.py new file mode 100644 index 0000000..ea96992 --- /dev/null +++ b/tests/test_pixiv_downloader.py @@ -0,0 +1,245 @@ +"""Unit tests for PixivDownloader — no network, no DB. + +The HTTP layer is stubbed via the `session` seam (a fake session whose `.get` +returns canned streaming bytes). Media items come from the real +PixivClient.extract_media over the shared fixture page, so the downloader is +exercised against the exact shapes the client produces. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import requests + +from backend.app.services.pixiv_client import MediaItem, PixivClient +from backend.app.services.pixiv_downloader import PixivDownloader + +_FIXTURE = Path(__file__).parent / "fixtures" / "pixiv_user_illusts_page1.json" + +# Minimal valid PNG so the file_validator passes on the happy path. +_PNG_HEAD = b"\x89PNG\r\n\x1a\n" +_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82" +_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL + + +class _FakeResponse: + def __init__(self, payload: bytes, status_code: int = 200, headers=None): + self._payload = payload + self.status_code = status_code + self.headers = headers or {} + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"HTTP {self.status_code}") + return None + + def iter_content(self, chunk_size=65536): + for i in range(0, len(self._payload), chunk_size): + yield self._payload[i : i + chunk_size] + + +class _FakeSession: + """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES).""" + + def __init__(self, payloads: dict[str, bytes] | None = None): + self.payloads = payloads or {} + self.calls: list[str] = [] + + def get(self, url, stream=False, timeout=None, headers=None): + self.calls.append(url) + return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) + + +def _client(): + # Parsing only — extract_media for non-ugoira works never issues a request. + return PixivClient("tok", session=_FakeSession()) + + +def _post_and_media(work_id): + page = json.loads(_FIXTURE.read_text()) + client = _client() + for work in page["illusts"]: + if work["id"] == work_id: + post = client._normalize(work) + return post, client.extract_media(post, {}) + raise AssertionError(f"no work {work_id} in fixture") + + +def _downloader(tmp_path, session=None): + return PixivDownloader( + tmp_path, session=session if session is not None else _FakeSession(), + ) + + +def test_download_post_writes_gallery_dl_layout(tmp_path): + post, media = _post_and_media(111) + dl = _downloader(tmp_path) + outcomes = dl.download_post(post, media, "artist-a") + + assert [o.status for o in outcomes] == ["downloaded", "downloaded"] + flat = tmp_path / "artist-a" / "pixiv" / "pixiv" + p0 = flat / "111_Multi Page Adventure_00.png" + p1 = flat / "111_Multi Page Adventure_01.png" + assert p0.is_file() + assert p1.is_file() + # No per-post directory — pixiv's layout is flat (gallery-dl parity). + assert {p.parent for p in (p0, p1)} == {flat} + + +def test_download_post_writes_minimal_sidecar(tmp_path): + post, media = _post_and_media(222) + dl = _downloader(tmp_path) + outcomes = dl.download_post(post, media, "artist-a") + + assert outcomes[0].status == "downloaded" + sidecar = outcomes[0].path.with_suffix(".json") + data = json.loads(sidecar.read_text()) + # Post-first: image identity ONLY — the body lives in the post record. + assert data == { + "category": "pixiv", + "id": "222", + "source_url": media[0].url, + } + + +def test_download_post_skips_seen(tmp_path): + post, media = _post_and_media(222) + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post(post, media, "artist-a", is_seen=lambda m: True) + assert [o.status for o in outcomes] == ["skipped_seen"] + assert session.calls == [] + + +def test_download_post_skips_on_disk(tmp_path): + post, media = _post_and_media(222) + flat = tmp_path / "artist-a" / "pixiv" / "pixiv" + flat.mkdir(parents=True) + existing = flat / "222_Single Piece_00.jpg" + existing.write_bytes(b"already here") + + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post(post, media, "artist-a") + assert [o.status for o in outcomes] == ["skipped_disk"] + assert outcomes[0].path == existing + assert session.calls == [] + assert existing.read_bytes() == b"already here" + + +def test_recapture_surfaces_on_disk_path_for_seen_media(tmp_path): + post, media = _post_and_media(222) + flat = tmp_path / "artist-a" / "pixiv" / "pixiv" + flat.mkdir(parents=True) + (flat / "222_Single Piece_00.jpg").write_bytes(b"kept") + + dl = _downloader(tmp_path) + outcomes = dl.download_post( + post, media, "artist-a", is_seen=lambda m: True, recapture=True, + ) + # On disk → surfaced with its path (relink channel); seen-but-missing + # stays skipped_seen (recovery's job, not recapture's). + assert outcomes[0].status == "skipped_disk" + assert outcomes[0].path is not None + + +def test_recapture_does_not_redownload_missing_seen_media(tmp_path): + post, media = _post_and_media(222) + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post( + post, media, "artist-a", is_seen=lambda m: True, recapture=True, + ) + assert [o.status for o in outcomes] == ["skipped_seen"] + assert session.calls == [] + + +def test_download_post_honours_should_stop(tmp_path): + post, media = _post_and_media(111) + dl = _downloader(tmp_path) + outcomes = dl.download_post(post, media, "artist-a", should_stop=lambda: True) + assert outcomes == [] + + +def test_filename_is_sanitized(tmp_path): + post, _ = _post_and_media(222) + weird = MediaItem( + url="https://i.pximg.net/img-original/img/2026/06/18/12/30/00/222_p0.jpg", + filename='222_What? A "Title"/Slash_00.jpg', + kind="image", + filehash=None, + post_id="222", + media_id="p0", + ) + dl = _downloader(tmp_path) + outcomes = dl.download_post(post, [weird], "artist-a") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.name == "222_What_ A _Title__Slash_00.jpg" + + +def test_own_session_carries_app_referer(tmp_path): + # Built session (no injection) must carry the app profile — i.pximg.net + # 403s GETs without the app-api Referer. + dl = PixivDownloader(tmp_path) + assert dl.session.headers["Referer"] == "https://app-api.pixiv.net/" + assert dl.session.headers["App-OS"] == "ios" + + +def test_write_post_record_enriched(tmp_path): + post, _ = _post_and_media(111) + dl = _downloader(tmp_path) + outcome = dl.write_post_record(post, "artist-a") + + assert outcome.path == ( + tmp_path / "artist-a" / "pixiv" / "pixiv" / "_post_111.json" + ) + assert outcome.title == "Multi Page Adventure" + assert outcome.post_type == "illust" + assert outcome.body_chars == len(post["attributes"]["content"]) + + data = json.loads(outcome.path.read_text()) + assert data["category"] == "pixiv" + assert data["id"] == "111" + assert data["published_at"] == "2026-06-20T18:00:00+09:00" + assert data["rating"] == "General" + assert data["page_count"] == 2 + assert data["total_bookmarks"] == 250 + assert data["total_view"] == 1000 + assert data["illust_ai_type"] == 1 + assert data["series"] == {"id": 4242, "title": "Adventure Series"} + assert data["tags"] == [ + {"name": "オリジナル", "translated_name": "original"}, + {"name": "女の子", "translated_name": "girl"}, + ] + assert data["user"] == {"id": 99, "account": "exartist", "name": "Example Artist"} + # JP tags stay human-readable on disk (ensure_ascii=False). + assert "オリジナル" in outcome.path.read_text() + + +def test_write_post_record_rating_r18(tmp_path): + post, _ = _post_and_media(222) + dl = _downloader(tmp_path) + outcome = dl.write_post_record(post, "artist-a") + data = json.loads(outcome.path.read_text()) + assert data["rating"] == "R-18" + assert data["is_bookmarked"] is True + + +def test_write_post_record_includes_ugoira_frames(tmp_path): + post, _ = _post_and_media(333) + frames = [{"file": "000000.jpg", "delay": 90}] + post["_work"]["_ugoira_frames"] = frames + dl = _downloader(tmp_path) + outcome = dl.write_post_record(post, "artist-a") + data = json.loads(outcome.path.read_text()) + assert data["type"] == "ugoira" + assert data["ugoira_frames"] == frames + + +def test_write_post_record_without_id(tmp_path): + dl = _downloader(tmp_path) + outcome = dl.write_post_record({"id": None, "attributes": {}}, "artist-a") + assert outcome.path is None + assert outcome.body_chars == 0