feat(pixiv): native downloader — gallery-dl layout parity + enriched post record (#129 step 2)

PixivDownloader writes originals to the exact pre-cutover gallery-dl layout
(<artist_slug>/pixiv/pixiv/{id}_{title[:50]}_{NN}.{ext} — flat, double
platform segment) so tier-2 disk-skip recognizes existing files. Post-first:
per-media sidecar is identity-only; the post record (_post_<id>.json — id
suffix because the flat layout would collide a bare _post.json) carries the
enrichment: tags + EN translations, rating from x_restrict, series,
view/bookmark/comment counts, AI flag, dimensions, author, and ugoira frame
delays (the zip has no timings). i.pximg.net media GETs ride the app-header
profile (403 without the app-api Referer).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

backend/app/services/pixiv_downloader.py

@@ -0,0 +1,239 @@
+"""Native Pixiv media downloader — the Pixiv counterpart to
+patreon_downloader / subscribestar_downloader.
+
+Given a normalized Pixiv work and its resolved `MediaItem`s
+(pixiv_client.extract_media), download the originals to gallery-dl's on-disk
+layout (so pre-cutover gallery-dl downloads are recognized on disk and not
+re-fetched), write the post-first sidecars the importer consumes, and report
+per-media outcomes.
+
+On-disk layout (matches FC's gallery-dl pixiv config, PLATFORM_DEFAULTS:
+base-directory `<images_root>/<artist_slug>/pixiv` + `directory:
+["{category}"]` + filename `{id}_{title[:50]}_{num:>02}.{extension}`):
+
+    <images_root>/<artist_slug>/pixiv/pixiv/<id>_<title50>_<NN>.<ext>
+
+— note the intentional DOUBLE `pixiv` segment: gallery-dl appended
+`{category}` under a base-directory that already ended in the platform name,
+and tier-2 disk-skip parity requires reproducing that exactly. The layout is
+FLAT (no per-post directory), so the post-first record is `_post_<id>.json`
+in the same directory (the id suffix prevents the collisions a bare
+`_post.json` would have here; phase 3 receives explicit post_record_paths, so
+the name is a convention, not a discovery key).
+
+Simpler than Patreon (no Mux/yt-dlp video branch) — the one special file is
+the ugoira frame zip, downloaded as-is; FC's archive-containment import
+extracts the frames, and the frame DELAYS ride the post record (the zip
+carries none — a future ugoira→video conversion needs them).
+
+PURE: no DB; the seen-skip is an injected predicate. FC runs on a plain-HTTP
+homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import requests
+
+from .native_ingest_common import (
+    BaseNativeDownloader,
+    MediaOutcome,
+    PostRecordOutcome,
+    sanitize_segment,
+)
+from .pixiv_client import PIXIV_APP_HEADERS, rating_label
+
+log = logging.getLogger(__name__)
+
+# Enrichment keys copied verbatim from the app-API work dict into the post
+# record (they're already JSON scalars/objects). Everything lands in
+# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and
+# structure without a schema change.
+_WORK_PASSTHROUGH_KEYS = (
+    "type",
+    "page_count",
+    "width",
+    "height",
+    "total_view",
+    "total_bookmarks",
+    "total_comments",
+    "is_bookmarked",
+    "illust_ai_type",
+    "series",
+)
+
+
+class PixivDownloader(BaseNativeDownloader):
+    """Download resolved Pixiv media to gallery-dl's on-disk layout.
+    Subclasses BaseNativeDownloader for the shared streaming GET
+    (transient-retry + Range-resume) and validation/quarantine. PURE: no DB."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        super().__init__(
+            images_root, cookies_path, platform="pixiv",
+            validate=validate, rate_limit=rate_limit, session=session,
+        )
+        if session is None:
+            # i.pximg.net 403s any GET without the app Referer; mirror the
+            # client's full app-header profile (gallery-dl serves media off
+            # the same session it drives the API with). An injected session
+            # (tests) owns its own headers.
+            self.session.headers.update(PIXIV_APP_HEADERS)
+
+    # -- public ------------------------------------------------------------
+
+    def download_post(
+        self,
+        post: dict,
+        media_items: list,
+        artist_slug: str,
+        *,
+        is_seen: Callable[[object], bool] = lambda m: False,
+        should_stop: Callable[[], bool] = lambda: False,
+        recapture: bool = False,
+    ) -> list[MediaOutcome]:
+        """Download every media item of one work; return per-item outcomes.
+        Mirrors SubscribeStarDownloader.download_post (two-tier skip, mid-post
+        time-box, recapture surfacing)."""
+        flat_dir = self._flat_dir(artist_slug)
+        outcomes: list[MediaOutcome] = []
+        for media in media_items:
+            if should_stop():
+                break
+            try:
+                outcomes.append(
+                    self._download_one(
+                        post, media, flat_dir, artist_slug, is_seen,
+                        recapture=recapture,
+                    )
+                )
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "Pixiv media failed (work %s, %s): %s",
+                    post.get("id"), getattr(media, "media_id", "?"), exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    def _flat_dir(self, artist_slug: str) -> Path:
+        # Double platform segment — gallery-dl layout parity (module docstring).
+        return self.images_root / artist_slug / "pixiv" / "pixiv"
+
+    # -- per-item ----------------------------------------------------------
+
+    def _download_one(
+        self,
+        post: dict,
+        media,
+        flat_dir: Path,
+        artist_slug: str,
+        is_seen: Callable[[object], bool],
+        *,
+        recapture: bool = False,
+    ) -> MediaOutcome:
+        seen = is_seen(media)
+        if seen and not recapture:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        # The client's filename already carries the {id}_{title50}_{NN} shape;
+        # only path-restrict sanitization happens here.
+        media_path = flat_dir / sanitize_segment(media.filename)
+
+        if media_path.exists():  # tier-2: already on disk
+            return MediaOutcome(
+                media=media, status="skipped_disk", path=media_path, error=None
+            )
+        # recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
+        if seen:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        flat_dir.mkdir(parents=True, exist_ok=True)
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+
+        out_path = self._fetch_get(media.url, media_path)
+        reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
+        if reason is not None:
+            return MediaOutcome(
+                media=media, status="quarantined", path=quarantine_dest, error=reason,
+            )
+        self._write_minimal_sidecar(post, out_path, source_url=media.url)
+        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
+
+    # -- post record ---------------------------------------------------------
+
+    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
+        """Write the post-first `_post_<id>.json` — the sole writer of the post
+        body/metadata on the native path. Beyond the standard body fields, the
+        record carries pixiv's own structure (tags + EN translations, rating,
+        series, view/bookmark counts, AI flag, dimensions, author, ugoira frame
+        delays) so the archive keeps what the platform knows about the work."""
+        attrs = post.get("attributes") or {}
+        work = post.get("_work") or {}
+        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
+        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
+        pid = str(post.get("id") or "")
+        if not pid:
+            return PostRecordOutcome(
+                path=None, post_type=post_type, title=title, body_chars=0,
+            )
+
+        content = attrs.get("content")
+        content = content if isinstance(content, str) else ""
+        data: dict = {
+            "category": "pixiv",
+            "id": pid,
+            "title": title or "",
+            "content": content,
+            "published_at": attrs.get("published_at"),
+            # The post permalink is synthesized by platforms/pixiv.py
+            # derive_post_url from `id` at parse time — no url key here.
+            "rating": rating_label(work.get("x_restrict")),
+        }
+        for key in _WORK_PASSTHROUGH_KEYS:
+            if key in work:
+                data[key] = work[key]
+        tags = work.get("tags")
+        if isinstance(tags, list):
+            data["tags"] = [
+                {
+                    "name": t.get("name"),
+                    "translated_name": t.get("translated_name"),
+                }
+                for t in tags
+                if isinstance(t, dict)
+            ]
+        user = work.get("user")
+        if isinstance(user, dict):
+            data["user"] = {
+                "id": user.get("id"),
+                "account": user.get("account"),
+                "name": user.get("name"),
+            }
+        # Memoized by extract_media's ugoira branch; the zip has no timings.
+        frames = work.get("_ugoira_frames")
+        if frames:
+            data["ugoira_frames"] = frames
+
+        flat_dir = self._flat_dir(artist_slug)
+        flat_dir.mkdir(parents=True, exist_ok=True)
+        path = flat_dir / f"_post_{pid}.json"
+        path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
+        return PostRecordOutcome(
+            path=path, post_type=post_type, title=title, body_chars=len(content),
+        )

tests/test_pixiv_downloader.py

@@ -0,0 +1,245 @@
+"""Unit tests for PixivDownloader — no network, no DB.
+
+The HTTP layer is stubbed via the `session` seam (a fake session whose `.get`
+returns canned streaming bytes). Media items come from the real
+PixivClient.extract_media over the shared fixture page, so the downloader is
+exercised against the exact shapes the client produces.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import requests
+
+from backend.app.services.pixiv_client import MediaItem, PixivClient
+from backend.app.services.pixiv_downloader import PixivDownloader
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "pixiv_user_illusts_page1.json"
+
+# Minimal valid PNG so the file_validator passes on the happy path.
+_PNG_HEAD = b"\x89PNG\r\n\x1a\n"
+_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
+_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL
+
+
+class _FakeResponse:
+    def __init__(self, payload: bytes, status_code: int = 200, headers=None):
+        self._payload = payload
+        self.status_code = status_code
+        self.headers = headers or {}
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code}")
+        return None
+
+    def iter_content(self, chunk_size=65536):
+        for i in range(0, len(self._payload), chunk_size):
+            yield self._payload[i : i + chunk_size]
+
+
+class _FakeSession:
+    """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES)."""
+
+    def __init__(self, payloads: dict[str, bytes] | None = None):
+        self.payloads = payloads or {}
+        self.calls: list[str] = []
+
+    def get(self, url, stream=False, timeout=None, headers=None):
+        self.calls.append(url)
+        return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
+
+
+def _client():
+    # Parsing only — extract_media for non-ugoira works never issues a request.
+    return PixivClient("tok", session=_FakeSession())
+
+
+def _post_and_media(work_id):
+    page = json.loads(_FIXTURE.read_text())
+    client = _client()
+    for work in page["illusts"]:
+        if work["id"] == work_id:
+            post = client._normalize(work)
+            return post, client.extract_media(post, {})
+    raise AssertionError(f"no work {work_id} in fixture")
+
+
+def _downloader(tmp_path, session=None):
+    return PixivDownloader(
+        tmp_path, session=session if session is not None else _FakeSession(),
+    )
+
+
+def test_download_post_writes_gallery_dl_layout(tmp_path):
+    post, media = _post_and_media(111)
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(post, media, "artist-a")
+
+    assert [o.status for o in outcomes] == ["downloaded", "downloaded"]
+    flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
+    p0 = flat / "111_Multi Page Adventure_00.png"
+    p1 = flat / "111_Multi Page Adventure_01.png"
+    assert p0.is_file()
+    assert p1.is_file()
+    # No per-post directory — pixiv's layout is flat (gallery-dl parity).
+    assert {p.parent for p in (p0, p1)} == {flat}
+
+
+def test_download_post_writes_minimal_sidecar(tmp_path):
+    post, media = _post_and_media(222)
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(post, media, "artist-a")
+
+    assert outcomes[0].status == "downloaded"
+    sidecar = outcomes[0].path.with_suffix(".json")
+    data = json.loads(sidecar.read_text())
+    # Post-first: image identity ONLY — the body lives in the post record.
+    assert data == {
+        "category": "pixiv",
+        "id": "222",
+        "source_url": media[0].url,
+    }
+
+
+def test_download_post_skips_seen(tmp_path):
+    post, media = _post_and_media(222)
+    session = _FakeSession()
+    dl = _downloader(tmp_path, session=session)
+    outcomes = dl.download_post(post, media, "artist-a", is_seen=lambda m: True)
+    assert [o.status for o in outcomes] == ["skipped_seen"]
+    assert session.calls == []
+
+
+def test_download_post_skips_on_disk(tmp_path):
+    post, media = _post_and_media(222)
+    flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
+    flat.mkdir(parents=True)
+    existing = flat / "222_Single Piece_00.jpg"
+    existing.write_bytes(b"already here")
+
+    session = _FakeSession()
+    dl = _downloader(tmp_path, session=session)
+    outcomes = dl.download_post(post, media, "artist-a")
+    assert [o.status for o in outcomes] == ["skipped_disk"]
+    assert outcomes[0].path == existing
+    assert session.calls == []
+    assert existing.read_bytes() == b"already here"
+
+
+def test_recapture_surfaces_on_disk_path_for_seen_media(tmp_path):
+    post, media = _post_and_media(222)
+    flat = tmp_path / "artist-a" / "pixiv" / "pixiv"
+    flat.mkdir(parents=True)
+    (flat / "222_Single Piece_00.jpg").write_bytes(b"kept")
+
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(
+        post, media, "artist-a", is_seen=lambda m: True, recapture=True,
+    )
+    # On disk → surfaced with its path (relink channel); seen-but-missing
+    # stays skipped_seen (recovery's job, not recapture's).
+    assert outcomes[0].status == "skipped_disk"
+    assert outcomes[0].path is not None
+
+
+def test_recapture_does_not_redownload_missing_seen_media(tmp_path):
+    post, media = _post_and_media(222)
+    session = _FakeSession()
+    dl = _downloader(tmp_path, session=session)
+    outcomes = dl.download_post(
+        post, media, "artist-a", is_seen=lambda m: True, recapture=True,
+    )
+    assert [o.status for o in outcomes] == ["skipped_seen"]
+    assert session.calls == []
+
+
+def test_download_post_honours_should_stop(tmp_path):
+    post, media = _post_and_media(111)
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(post, media, "artist-a", should_stop=lambda: True)
+    assert outcomes == []
+
+
+def test_filename_is_sanitized(tmp_path):
+    post, _ = _post_and_media(222)
+    weird = MediaItem(
+        url="https://i.pximg.net/img-original/img/2026/06/18/12/30/00/222_p0.jpg",
+        filename='222_What? A "Title"/Slash_00.jpg',
+        kind="image",
+        filehash=None,
+        post_id="222",
+        media_id="p0",
+    )
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(post, [weird], "artist-a")
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.name == "222_What_ A _Title__Slash_00.jpg"
+
+
+def test_own_session_carries_app_referer(tmp_path):
+    # Built session (no injection) must carry the app profile — i.pximg.net
+    # 403s GETs without the app-api Referer.
+    dl = PixivDownloader(tmp_path)
+    assert dl.session.headers["Referer"] == "https://app-api.pixiv.net/"
+    assert dl.session.headers["App-OS"] == "ios"
+
+
+def test_write_post_record_enriched(tmp_path):
+    post, _ = _post_and_media(111)
+    dl = _downloader(tmp_path)
+    outcome = dl.write_post_record(post, "artist-a")
+
+    assert outcome.path == (
+        tmp_path / "artist-a" / "pixiv" / "pixiv" / "_post_111.json"
+    )
+    assert outcome.title == "Multi Page Adventure"
+    assert outcome.post_type == "illust"
+    assert outcome.body_chars == len(post["attributes"]["content"])
+
+    data = json.loads(outcome.path.read_text())
+    assert data["category"] == "pixiv"
+    assert data["id"] == "111"
+    assert data["published_at"] == "2026-06-20T18:00:00+09:00"
+    assert data["rating"] == "General"
+    assert data["page_count"] == 2
+    assert data["total_bookmarks"] == 250
+    assert data["total_view"] == 1000
+    assert data["illust_ai_type"] == 1
+    assert data["series"] == {"id": 4242, "title": "Adventure Series"}
+    assert data["tags"] == [
+        {"name": "オリジナル", "translated_name": "original"},
+        {"name": "女の子", "translated_name": "girl"},
+    ]
+    assert data["user"] == {"id": 99, "account": "exartist", "name": "Example Artist"}
+    # JP tags stay human-readable on disk (ensure_ascii=False).
+    assert "オリジナル" in outcome.path.read_text()
+
+
+def test_write_post_record_rating_r18(tmp_path):
+    post, _ = _post_and_media(222)
+    dl = _downloader(tmp_path)
+    outcome = dl.write_post_record(post, "artist-a")
+    data = json.loads(outcome.path.read_text())
+    assert data["rating"] == "R-18"
+    assert data["is_bookmarked"] is True
+
+
+def test_write_post_record_includes_ugoira_frames(tmp_path):
+    post, _ = _post_and_media(333)
+    frames = [{"file": "000000.jpg", "delay": 90}]
+    post["_work"]["_ugoira_frames"] = frames
+    dl = _downloader(tmp_path)
+    outcome = dl.write_post_record(post, "artist-a")
+    data = json.loads(outcome.path.read_text())
+    assert data["type"] == "ugoira"
+    assert data["ugoira_frames"] == frames
+
+
+def test_write_post_record_without_id(tmp_path):
+    dl = _downloader(tmp_path)
+    outcome = dl.write_post_record({"id": None, "attributes": {}}, "artist-a")
+    assert outcome.path is None
+    assert outcome.body_chars == 0