feat(pixiv): ledger models + migration 0076 + PixivIngester adapter (#129 step 3)

pixiv_seen_media / pixiv_failed_media mirror the Patreon/SubscribeStar
ledgers (keys are always synthesized <illust_id>:p<num> / <illust_id>:ugoira
— pximg URLs carry no content hash). PixivIngester wires client/downloader/
ledgers into ingest_core with drift label 'Pixiv app API' and the new
body_canary=False opt-out: caption-less pixiv artists are common, so the
zero-bodies #862 alarm would false-positive here — the client's
response-shape drift checks cover that failure class instead. auth_token
joins the uniform adapter constructor (pixiv is the first token-auth native
platform). verify_pixiv_credential = one OAuth refresh, no feed walk.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

backend/app/models/__init__.py

@@ -23,6 +23,8 @@ from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
 from .patreon_failed_media import PatreonFailedMedia
 from .patreon_seen_media import PatreonSeenMedia
+from .pixiv_failed_media import PixivFailedMedia
+from .pixiv_seen_media import PixivSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
 from .series_chapter import SeriesChapter
@@ -48,6 +50,8 @@ __all__ = [
     "Credential",
     "PatreonFailedMedia",
     "PatreonSeenMedia",
+    "PixivFailedMedia",
+    "PixivSeenMedia",
     "SubscribeStarFailedMedia",
     "SubscribeStarSeenMedia",
     "Post",

backend/app/models/pixiv_failed_media.py

@@ -0,0 +1,45 @@
+"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
+failing to download/validate.
+
+Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
+walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
+otherwise re-error forever and re-burn backfill chunks. After ``attempts``
+reaches the dead-letter threshold the ingester skips it on routine
+tick/backfill walks (recovery still re-attempts). A later clean download
+clears the row.
+
+`filehash` is the same synthesized ``<illust_id>:p<num>`` /
+``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
+is the upsert key.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PixivFailedMedia(Base):
+    __tablename__ = "pixiv_failed_media"
+    __table_args__ = (
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    first_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/pixiv_seen_media.py

@@ -0,0 +1,42 @@
+"""PixivSeenMedia — per-source ledger of Pixiv media already
+downloaded+processed.
+
+Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
+ingester (replacing gallery-dl). One queryable row per (source, media) so
+routine walks skip media we've already ingested; recovery mode bypasses the
+ledger to re-walk.
+
+Pixiv original URLs carry no content hash, so `filehash` is always the
+synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
+zip) key — stable across any URL-shape drift. String(128) matches the sibling
+ledgers.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PixivSeenMedia(Base):
+    __tablename__ = "pixiv_seen_media"
+    __table_args__ = (
+        # Dedup key the downloader upserts against: one ledger row per
+        # (source, media). A second sighting of the same media is a no-op.
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    seen_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/services/ingest_core.py

@@ -90,6 +90,7 @@ class Ingester:
         platform: str,
         error_base: type[Exception],
         drift_label: str | None = None,
+        body_canary: bool = True,
     ):
         self.client = client
         self.downloader = downloader
@@ -105,6 +106,12 @@ class Ingester:
         # update"). Defaults to the platform name; adapters pass a richer phrase
         # (e.g. "Patreon API", "SubscribeStar markup").
         self._drift_label = drift_label or platform
+        # #862 canary opt-out: platforms whose posts legitimately have empty
+        # bodies across large samples (pixiv — caption-less artists are common)
+        # would false-positive the zero-bodies-means-drift alarm; their clients
+        # catch drift structurally (response-shape checks) instead. The
+        # "bodies X/N" summary line still surfaces the ratio either way.
+        self._body_canary = body_canary
 
     # -- public ------------------------------------------------------------
 
@@ -536,7 +543,11 @@ class Ingester:
         # creds") so the breakage screams instead of silently archiving empties.
         # Only reached on an otherwise-clean walk (timeout/stop/error returned
         # above), so it never masks a more specific failure.
-        if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0:
+        if (
+            self._body_canary
+            and posts_recorded >= _CANARY_MIN_SAMPLE
+            and posts_with_body == 0
+        ):
             msg = (
                 f"Post-body canary: extracted a body from 0 of {posts_recorded} "
                 "posts — Patreon's body field shape likely changed; the ingester "

backend/app/services/pixiv_ingester.py

@@ -0,0 +1,117 @@
+"""Native Pixiv ingester — the Pixiv ADAPTER over the platform-agnostic core
+(`ingest_core.Ingester`).
+
+Thin counterpart to patreon_ingester / subscribestar_ingester: wires the Pixiv
+client/downloader/ledger models/constraints/key into the core and supplies the
+Pixiv failure mapping. The modes (tick / backfill / recovery / recapture), the
+seen + dead-letter ledgers, cursor checkpointing, and the post-first capture
+all live in the core. `download_service.download_source` drives
+`PixivIngester.run` exactly as it drives the other two.
+
+`campaign_id` is the numeric pixiv user id (download_backends extracts it from
+the source URL — no network resolver). Auth is the operator's OAuth refresh
+token (the token-type Credential), passed as `auth_token` — pixiv is the first
+native platform authenticating by token rather than cookies, so the uniform
+constructor accepts both and ignores what it doesn't need.
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from ..models import PixivFailedMedia, PixivSeenMedia
+from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
+from .pixiv_client import MediaItem, PixivAPIError, PixivClient
+from .pixiv_downloader import PixivDownloader
+
+__all__ = [
+    "DEAD_LETTER_THRESHOLD",
+    "PixivIngester",
+    "_ledger_key",
+    "verify_pixiv_credential",
+]
+
+log = logging.getLogger(__name__)
+
+_LEDGER_KEY_MAX = 128
+
+
+def _ledger_key(media: MediaItem) -> str:
+    """Stable per-media identity for the cross-run seen-ledger. Pixiv original
+    URLs carry no content hash, so the key is the page/zip identity scoped to
+    its work: `<illust_id>:p<num>` / `<illust_id>:ugoira`. Bounded to the
+    column width."""
+    if media.filehash:
+        return media.filehash
+    return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
+
+
+class PixivIngester(Ingester):
+    """Walk a pixiv user's works, download unseen originals, return a
+    `DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
+    `downloader` are injectable seams so unit tests run without network."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None,
+        session_factory: Callable[[], object],
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        request_sleep: float = 0.0,
+        auth_token: str | None = None,
+        client: PixivClient | None = None,
+        downloader: PixivDownloader | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        resolved_client = (
+            client
+            if client is not None
+            else PixivClient(auth_token, request_sleep=request_sleep)
+        )
+        resolved_downloader = (
+            downloader
+            if downloader is not None
+            else PixivDownloader(
+                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
+            )
+        )
+        super().__init__(
+            client=resolved_client,
+            downloader=resolved_downloader,
+            session_factory=session_factory,
+            seen_model=PixivSeenMedia,
+            failed_model=PixivFailedMedia,
+            seen_constraint="uq_pixiv_seen_media_source_id",
+            failed_constraint="uq_pixiv_failed_media_source_id",
+            ledger_key=_ledger_key,
+            platform="pixiv",
+            error_base=PixivAPIError,
+            # API_DRIFT message phrasing; the base Ingester._failure_result owns
+            # the auth/drift/HTTP→error_type mapping (shared across platforms).
+            drift_label="Pixiv app API",
+            # Captions are legitimately empty for many pixiv artists, so the
+            # zero-bodies #862 canary would false-positive here; the client's
+            # response-shape checks (missing `illusts` → drift) cover the same
+            # failure class structurally.
+            body_canary=False,
+        )
+
+
+async def verify_pixiv_credential(
+    auth_token: str | None,
+) -> tuple[bool | None, str]:
+    """Native Pixiv credential probe — one OAuth refresh via
+    PixivClient.verify_auth (the exact call that fails when the token is
+    bad; no feed walk). Returns the uniform `(ok, message)` contract so
+    download_backends.verify_source_credential treats it like the others."""
+    client = PixivClient(auth_token)
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, client.verify_auth)