feat(pixiv): ledger models + migration 0076 + PixivIngester adapter (#129 step 3)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Failing after 31s
CI / integration (push) Successful in 3m36s

pixiv_seen_media / pixiv_failed_media mirror the Patreon/SubscribeStar
ledgers (keys are always synthesized <illust_id>:p<num> / <illust_id>:ugoira
— pximg URLs carry no content hash). PixivIngester wires client/downloader/
ledgers into ingest_core with drift label 'Pixiv app API' and the new
body_canary=False opt-out: caption-less pixiv artists are common, so the
zero-bodies #862 alarm would false-positive here — the client's
response-shape drift checks cover that failure class instead. auth_token
joins the uniform adapter constructor (pixiv is the first token-auth native
platform). verify_pixiv_credential = one OAuth refresh, no feed walk.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
2026-07-03 09:45:14 -04:00
parent 7ef2ecd82f
commit 0563b2d750
7 changed files with 607 additions and 1 deletions
+4
View File
@@ -23,6 +23,8 @@ from .library_audit_run import LibraryAuditRun
from .ml_settings import MLSettings
from .patreon_failed_media import PatreonFailedMedia
from .patreon_seen_media import PatreonSeenMedia
from .pixiv_failed_media import PixivFailedMedia
from .pixiv_seen_media import PixivSeenMedia
from .post import Post
from .post_attachment import PostAttachment
from .series_chapter import SeriesChapter
@@ -48,6 +50,8 @@ __all__ = [
"Credential",
"PatreonFailedMedia",
"PatreonSeenMedia",
"PixivFailedMedia",
"PixivSeenMedia",
"SubscribeStarFailedMedia",
"SubscribeStarSeenMedia",
"Post",
+45
View File
@@ -0,0 +1,45 @@
"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
failing to download/validate.
Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
otherwise re-error forever and re-burn backfill chunks. After ``attempts``
reaches the dead-letter threshold the ingester skips it on routine
tick/backfill walks (recovery still re-attempts). A later clean download
clears the row.
`filehash` is the same synthesized ``<illust_id>:p<num>`` /
``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
is the upsert key.
"""
from datetime import datetime
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import DateTime
from .base import Base
class PixivFailedMedia(Base):
__tablename__ = "pixiv_failed_media"
__table_args__ = (
UniqueConstraint(
"source_id", "filehash", name="uq_pixiv_failed_media_source_id"
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_id: Mapped[int] = mapped_column(
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
)
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
first_failed_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
last_failed_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
+42
View File
@@ -0,0 +1,42 @@
"""PixivSeenMedia — per-source ledger of Pixiv media already
downloaded+processed.
Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
ingester (replacing gallery-dl). One queryable row per (source, media) so
routine walks skip media we've already ingested; recovery mode bypasses the
ledger to re-walk.
Pixiv original URLs carry no content hash, so `filehash` is always the
synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
zip) key — stable across any URL-shape drift. String(128) matches the sibling
ledgers.
"""
from datetime import datetime
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import DateTime
from .base import Base
class PixivSeenMedia(Base):
__tablename__ = "pixiv_seen_media"
__table_args__ = (
# Dedup key the downloader upserts against: one ledger row per
# (source, media). A second sighting of the same media is a no-op.
UniqueConstraint(
"source_id", "filehash", name="uq_pixiv_seen_media_source_id"
),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_id: Mapped[int] = mapped_column(
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
)
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
seen_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)