bvandeusen
f678819093
Phase 1, step 1 of moving SubscribeStar off gallery-dl onto the native core ingester (milestone: SubscribeStar native). Mirror of the Patreon ledger: SubscribeStarSeenMedia (skip already-ingested media on routine walks; recovery bypasses) and SubscribeStarFailedMedia (dead-letter so persistently-failing media stops re-burning backfill chunks). Per operator decision, dedicated per-platform tables (not a generalized shared ledger). filehash is String(128): a CDN content hash when the URL carries one, else a synthesized <post_id>:<filename> key. UNIQUE (source_id, filehash) upsert key. Registered in models/__init__; migration 0054 creates both tables (down 0053). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
83 lines
2.7 KiB
Python
83 lines
2.7 KiB
Python
"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers
|
|
|
|
Revision ID: 0054
|
|
Revises: 0053
|
|
Create Date: 2026-06-17
|
|
|
|
SubscribeStar native ingester (phase 1 of the gallery-dl → native-core
|
|
migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so
|
|
routine walks skip already-ingested media (recovery bypasses it) and a
|
|
dead-letter ledger so persistently-failing media stops re-burning backfill
|
|
chunks. `filehash` is a CDN content hash when present, else a synthesized
|
|
``<post_id>:<filename>`` key — hence String(128). UNIQUE (source_id, filehash)
|
|
is the upsert key on each.
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision: str = "0054"
|
|
down_revision: Union[str, None] = "0053"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"subscribestar_seen_media",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column(
|
|
"source_id",
|
|
sa.Integer,
|
|
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
),
|
|
sa.Column("filehash", sa.String(128), nullable=False),
|
|
sa.Column("post_id", sa.String(64), nullable=True),
|
|
sa.Column(
|
|
"seen_at",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("NOW()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"source_id", "filehash", name="uq_subscribestar_seen_media_source_id"
|
|
),
|
|
)
|
|
op.create_table(
|
|
"subscribestar_failed_media",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column(
|
|
"source_id",
|
|
sa.Integer,
|
|
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
),
|
|
sa.Column("filehash", sa.String(128), nullable=False),
|
|
sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
|
|
sa.Column("last_error", sa.Text, nullable=True),
|
|
sa.Column(
|
|
"first_failed_at",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("NOW()"),
|
|
),
|
|
sa.Column(
|
|
"last_failed_at",
|
|
sa.DateTime(timezone=True),
|
|
nullable=False,
|
|
server_default=sa.text("NOW()"),
|
|
),
|
|
sa.UniqueConstraint(
|
|
"source_id", "filehash", name="uq_subscribestar_failed_media_source_id"
|
|
),
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("subscribestar_failed_media")
|
|
op.drop_table("subscribestar_seen_media")
|