feat(patreon): seen-ledger table + model — ingester build step 2a (plan #697)
patreon_seen_media(source_id, filehash, post_id, seen_at), UNIQUE(source_id, filehash) — our own queryable replacement for gallery-dl's archive.sqlite3. Routine walks skip seen media; recovery mode bypasses the ledger. filehash is a 32-hex CDN MD5 or a video:<post>:<media> sentinel (String(128)). alembic 0037 (← 0036). Integration test covers dedup + savepoint recovery. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
"""patreon_seen_media: per-source ledger of already-ingested Patreon media
|
||||
|
||||
Revision ID: 0037
|
||||
Revises: 0036
|
||||
Create Date: 2026-06-05
|
||||
|
||||
Native Patreon ingester (build step 2a). Replaces gallery-dl's
|
||||
archive.sqlite3 with our own queryable table. The downloader upserts one
|
||||
row per (source, media) so routine walks skip media we've already
|
||||
processed; a future "recovery" mode bypasses the ledger to re-walk.
|
||||
|
||||
`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
|
||||
``video:<post_id>:<media_id>`` — hence String(128). The unique
|
||||
constraint on (source_id, filehash) is the dedup upsert key.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0037"
|
||||
down_revision: Union[str, None] = "0036"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"patreon_seen_media",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"source_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("filehash", sa.String(128), nullable=False),
|
||||
sa.Column("post_id", sa.String(64), nullable=True),
|
||||
sa.Column(
|
||||
"seen_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("NOW()"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_patreon_seen_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("patreon_seen_media")
|
||||
Reference in New Issue
Block a user