bvandeusen
6222928746
patreon_seen_media(source_id, filehash, post_id, seen_at), UNIQUE(source_id, filehash) — our own queryable replacement for gallery-dl's archive.sqlite3. Routine walks skip seen media; recovery mode bypasses the ledger. filehash is a 32-hex CDN MD5 or a video:<post>:<media> sentinel (String(128)). alembic 0037 (← 0036). Integration test covers dedup + savepoint recovery. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
"""patreon_seen_media: per-source ledger of already-ingested Patreon media
|
|
|
|
Revision ID: 0037
|
|
Revises: 0036
|
|
Create Date: 2026-06-05
|
|
|
|
Native Patreon ingester (build step 2a). Replaces gallery-dl's
|
|
archive.sqlite3 with our own queryable table. The downloader upserts one
|
|
row per (source, media) so routine walks skip media we've already
|
|
processed; a future "recovery" mode bypasses the ledger to re-walk.
|
|
|
|
`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
|
|
``video:<post_id>:<media_id>`` — hence String(128). The unique
|
|
constraint on (source_id, filehash) is the dedup upsert key.
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision: str = "0037"
|
|
down_revision: Union[str, None] = "0036"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"patreon_seen_media",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column(
|
|
"source_id",
|
|
sa.Integer,
|
|
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
),
|
|
sa.Column("filehash", sa.String(128), nullable=False),
|
|
sa.Column("post_id", sa.String(64), nullable=True),
|
|
sa.Column(
|
|
"seen_at",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("NOW()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"source_id", "filehash", name="uq_patreon_seen_media_source_id"
|
|
),
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("patreon_seen_media")
|