diff --git a/alembic/versions/0037_patreon_seen_media.py b/alembic/versions/0037_patreon_seen_media.py new file mode 100644 index 0000000..255484e --- /dev/null +++ b/alembic/versions/0037_patreon_seen_media.py @@ -0,0 +1,53 @@ +"""patreon_seen_media: per-source ledger of already-ingested Patreon media + +Revision ID: 0037 +Revises: 0036 +Create Date: 2026-06-05 + +Native Patreon ingester (build step 2a). Replaces gallery-dl's +archive.sqlite3 with our own queryable table. The downloader upserts one +row per (source, media) so routine walks skip media we've already +processed; a future "recovery" mode bypasses the ledger to re-walk. + +`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form +``video::`` — hence String(128). The unique +constraint on (source_id, filehash) is the dedup upsert key. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0037" +down_revision: Union[str, None] = "0036" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_seen_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_seen_media") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 137311c..1445788 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -14,6 +14,7 @@ from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun from .ml_settings import MLSettings +from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment from .series_page import SeriesPage @@ -33,6 +34,7 @@ __all__ = [ "BackupRun", "Source", "Credential", + "PatreonSeenMedia", "Post", "PostAttachment", "SeriesPage", diff --git a/backend/app/models/patreon_seen_media.py b/backend/app/models/patreon_seen_media.py new file mode 100644 index 0000000..9ba92f2 --- /dev/null +++ b/backend/app/models/patreon_seen_media.py @@ -0,0 +1,38 @@ +"""PatreonSeenMedia — per-source ledger of Patreon media already +downloaded+processed. + +Replaces gallery-dl's archive.sqlite3 with our own queryable table so +routine walks can skip media we've already ingested (and a future +"recovery" mode can deliberately bypass the ledger to re-walk). + +`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos — +which have no stable content hash at discovery time — use a sentinel of +the form ``video::``, hence String(128) rather than 32. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class PatreonSeenMedia(Base): + __tablename__ = "patreon_seen_media" + __table_args__ = ( + # Dedup key the downloader upserts against: one ledger row per + # (source, media). A second sighting of the same media is a no-op. + UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + post_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/tests/test_patreon_seen_media.py b/tests/test_patreon_seen_media.py new file mode 100644 index 0000000..e80e9bf --- /dev/null +++ b/tests/test_patreon_seen_media.py @@ -0,0 +1,73 @@ +import pytest +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError + +from backend.app.models import Artist, PatreonSeenMedia, Source + +pytestmark = pytest.mark.integration + + +async def _source(db): + artist = Artist(name="Seen Ledger Artist", slug="seen-ledger-artist") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, + platform="patreon", + url="https://www.patreon.com/seenledger", + ) + db.add(source) + await db.flush() + return source.id + + +@pytest.mark.asyncio +async def test_patreon_seen_media_dedup(db): + source_id = await _source(db) + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="123456", + ) + ) + await db.flush() + + seen_at = await db.scalar( + select(PatreonSeenMedia.seen_at).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert seen_at is not None + + # A second sighting of the same (source_id, filehash) is rejected by the + # unique constraint. Wrap in a savepoint so the IntegrityError doesn't + # poison the outer transaction. + with pytest.raises(IntegrityError): + async with db.begin_nested(): + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="654321", + ) + ) + await db.flush() + + # Same filehash but a different post_id sentinel still inserts: the + # dedup axis is (source_id, filehash), not post_id. + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="video:123456:7890", + post_id="123456", + ) + ) + await db.flush() + + count = await db.scalar( + select(func.count(PatreonSeenMedia.id)).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert count == 2