feat(patreon): seen-ledger table + model — ingester build step 2a (plan #697)
patreon_seen_media(source_id, filehash, post_id, seen_at), UNIQUE(source_id, filehash) — our own queryable replacement for gallery-dl's archive.sqlite3. Routine walks skip seen media; recovery mode bypasses the ledger. filehash is a 32-hex CDN MD5 or a video:<post>:<media> sentinel (String(128)). alembic 0037 (← 0036). Integration test covers dedup + savepoint recovery. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ from .import_settings import ImportSettings
|
||||
from .import_task import ImportTask
|
||||
from .library_audit_run import LibraryAuditRun
|
||||
from .ml_settings import MLSettings
|
||||
from .patreon_seen_media import PatreonSeenMedia
|
||||
from .post import Post
|
||||
from .post_attachment import PostAttachment
|
||||
from .series_page import SeriesPage
|
||||
@@ -33,6 +34,7 @@ __all__ = [
|
||||
"BackupRun",
|
||||
"Source",
|
||||
"Credential",
|
||||
"PatreonSeenMedia",
|
||||
"Post",
|
||||
"PostAttachment",
|
||||
"SeriesPage",
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""PatreonSeenMedia — per-source ledger of Patreon media already
|
||||
downloaded+processed.
|
||||
|
||||
Replaces gallery-dl's archive.sqlite3 with our own queryable table so
|
||||
routine walks can skip media we've already ingested (and a future
|
||||
"recovery" mode can deliberately bypass the ledger to re-walk).
|
||||
|
||||
`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos —
|
||||
which have no stable content hash at discovery time — use a sentinel of
|
||||
the form ``video:<post_id>:<media_id>``, hence String(128) rather than 32.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PatreonSeenMedia(Base):
|
||||
__tablename__ = "patreon_seen_media"
|
||||
__table_args__ = (
|
||||
# Dedup key the downloader upserts against: one ledger row per
|
||||
# (source, media). A second sighting of the same media is a no-op.
|
||||
UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
Reference in New Issue
Block a user