bvandeusen
6222928746
patreon_seen_media(source_id, filehash, post_id, seen_at), UNIQUE(source_id, filehash) — our own queryable replacement for gallery-dl's archive.sqlite3. Routine walks skip seen media; recovery mode bypasses the ledger. filehash is a 32-hex CDN MD5 or a video:<post>:<media> sentinel (String(128)). alembic 0037 (← 0036). Integration test covers dedup + savepoint recovery. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
39 lines
1.5 KiB
Python
39 lines
1.5 KiB
Python
"""PatreonSeenMedia — per-source ledger of Patreon media already
|
|
downloaded+processed.
|
|
|
|
Replaces gallery-dl's archive.sqlite3 with our own queryable table so
|
|
routine walks can skip media we've already ingested (and a future
|
|
"recovery" mode can deliberately bypass the ledger to re-walk).
|
|
|
|
`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos —
|
|
which have no stable content hash at discovery time — use a sentinel of
|
|
the form ``video:<post_id>:<media_id>``, hence String(128) rather than 32.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
from sqlalchemy.types import DateTime
|
|
|
|
from .base import Base
|
|
|
|
|
|
class PatreonSeenMedia(Base):
|
|
__tablename__ = "patreon_seen_media"
|
|
__table_args__ = (
|
|
# Dedup key the downloader upserts against: one ledger row per
|
|
# (source, media). A second sighting of the same media is a no-op.
|
|
UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
source_id: Mapped[int] = mapped_column(
|
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
|
)
|
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
|
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
seen_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
)
|