diff --git a/alembic/versions/0054_subscribestar_ledgers.py b/alembic/versions/0054_subscribestar_ledgers.py new file mode 100644 index 0000000..59972ae --- /dev/null +++ b/alembic/versions/0054_subscribestar_ledgers.py @@ -0,0 +1,82 @@ +"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers + +Revision ID: 0054 +Revises: 0053 +Create Date: 2026-06-17 + +SubscribeStar native ingester (phase 1 of the gallery-dl → native-core +migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so +routine walks skip already-ingested media (recovery bypasses it) and a +dead-letter ledger so persistently-failing media stops re-burning backfill +chunks. `filehash` is a CDN content hash when present, else a synthesized +``:`` key — hence String(128). UNIQUE (source_id, filehash) +is the upsert key on each. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0054" +down_revision: Union[str, None] = "0053" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "subscribestar_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_seen_media_source_id" + ), + ) + op.create_table( + "subscribestar_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("subscribestar_failed_media") + op.drop_table("subscribestar_seen_media") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 4433dc2..6061f31 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -24,6 +24,8 @@ from .series_chapter import SeriesChapter from .series_page import SeriesPage from .series_suggestion import SeriesSuggestion from .source import Source +from .subscribestar_failed_media import SubscribeStarFailedMedia +from .subscribestar_seen_media import SubscribeStarSeenMedia from .tag import Tag, TagKind, image_tag from .tag_alias import TagAlias from .tag_allowlist import TagAllowlist @@ -41,6 +43,8 @@ __all__ = [ "Credential", "PatreonFailedMedia", "PatreonSeenMedia", + "SubscribeStarFailedMedia", + "SubscribeStarSeenMedia", "Post", "PostAttachment", "SeriesChapter", diff --git a/backend/app/models/subscribestar_failed_media.py b/backend/app/models/subscribestar_failed_media.py new file mode 100644 index 0000000..9201aa7 --- /dev/null +++ b/backend/app/models/subscribestar_failed_media.py @@ -0,0 +1,44 @@ +"""SubscribeStarFailedMedia — per-source dead-letter ledger of SubscribeStar +media that keeps failing to download/validate. + +Mirror of PatreonFailedMedia. Media that fails every walk (404'd CDN URL, +deleted post, persistently-corrupt bytes) would otherwise re-error forever and +re-burn backfill chunks. After ``attempts`` reaches the dead-letter threshold +the ingester skips it on routine tick/backfill walks (recovery still +re-attempts). A later clean download clears the row. + +`filehash` is the same per-media key the seen-ledger uses (CDN content hash or a +synthesized ``:`` key) — hence String(128). UNIQUE +(source_id, filehash) is the upsert key. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class SubscribeStarFailedMedia(Base): + __tablename__ = "subscribestar_failed_media" + __table_args__ = ( + UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_failed_media_source_id" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + first_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + last_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/subscribestar_seen_media.py b/backend/app/models/subscribestar_seen_media.py new file mode 100644 index 0000000..67b1608 --- /dev/null +++ b/backend/app/models/subscribestar_seen_media.py @@ -0,0 +1,40 @@ +"""SubscribeStarSeenMedia — per-source ledger of SubscribeStar media already +downloaded+processed. + +Mirror of PatreonSeenMedia for the SubscribeStar native ingester (replacing +gallery-dl). One queryable row per (source, media) so routine walks skip media +we've already ingested; recovery mode bypasses the ledger to re-walk. + +`filehash` is a CDN content hash when the media URL carries one, else a +synthesized ``:`` key (SubscribeStar URLs aren't always +content-addressed) — hence String(128) rather than 32. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class SubscribeStarSeenMedia(Base): + __tablename__ = "subscribestar_seen_media" + __table_args__ = ( + # Dedup key the downloader upserts against: one ledger row per + # (source, media). A second sighting of the same media is a no-op. + UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_seen_media_source_id" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + post_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + )