feat: define unified schema (Artist, Source, Post, ImageRecord, etc.) and initial migration

Implements the data model from spec §3 in one go so FC-2/FC-3 don't need
schema-adding migrations of their own. Artist is the unified entity for
both gallery 'artist:' tags and GallerySubscriber Subscriptions
(is_subscription flag). ImageProvenance is many-to-one, enabling the
enrich-on-duplicate rule for downloaded content that pHash-matches an
existing record.

The SigLIP embedding column uses pgvector(1152) for SigLIP-so400m;
swapping models in FC-2 will require a column-width migration.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-14 07:34:29 -04:00
parent a03655039c
commit d6a156dcd2
11 changed files with 619 additions and 3 deletions
@@ -0,0 +1,277 @@
"""initial unified schema
Revision ID: 0001
Revises:
Create Date: 2026-05-13
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from pgvector.sqlalchemy import Vector
revision: str = "0001"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
op.create_table(
"artist",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("slug", sa.String(length=255), nullable=False),
sa.Column("notes", sa.Text(), nullable=True),
sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("check_interval_seconds", sa.Integer(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.PrimaryKeyConstraint("id", name="pk_artist"),
sa.UniqueConstraint("name", name="uq_artist_name"),
sa.UniqueConstraint("slug", name="uq_artist_slug"),
)
op.create_table(
"source",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("artist_id", sa.Integer(), nullable=False),
sa.Column("platform", sa.String(length=64), nullable=False),
sa.Column("url", sa.Text(), nullable=False),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
sa.Column("config_overrides", sa.JSON(), nullable=True),
sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("check_interval_override", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("id", name="pk_source"),
)
op.create_index("ix_source_artist_id", "source", ["artist_id"])
op.create_table(
"credential",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("platform", sa.String(length=64), nullable=False),
sa.Column("kind", sa.String(length=32), nullable=False),
sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="active"),
sa.Column(
"captured_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint("id", name="pk_credential"),
sa.UniqueConstraint("platform", name="uq_credential_platform"),
)
op.create_table(
"post",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("external_post_id", sa.String(length=128), nullable=False),
sa.Column("post_url", sa.Text(), nullable=True),
sa.Column("post_title", sa.Text(), nullable=True),
sa.Column("post_date", sa.DateTime(timezone=True), nullable=True),
sa.Column("raw_metadata", sa.JSON(), nullable=True),
sa.Column(
"downloaded_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("id", name="pk_post"),
sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
)
op.create_index("ix_post_source_id", "post", ["source_id"])
op.create_table(
"image_record",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("sha256", sa.String(length=64), nullable=False),
sa.Column("phash", sa.String(length=32), nullable=True),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("mime", sa.String(length=64), nullable=False),
sa.Column("width", sa.Integer(), nullable=True),
sa.Column("height", sa.Integer(), nullable=True),
sa.Column("thumbnail_path", sa.Text(), nullable=True),
sa.Column(
"origin",
sa.Enum(
"downloaded",
"imported_filesystem",
"uploaded",
name="origin_enum",
),
nullable=False,
),
sa.Column("primary_post_id", sa.Integer(), nullable=True),
sa.Column("wd14_predictions", sa.JSON(), nullable=True),
sa.Column("wd14_model_version", sa.String(length=128), nullable=True),
sa.Column("siglip_embedding", Vector(1152), nullable=True),
sa.Column("siglip_model_version", sa.String(length=128), nullable=True),
sa.Column("centroid_scores", sa.JSON(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["primary_post_id"],
["post.id"],
name="fk_image_record_primary_post_id_post",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_image_record"),
sa.UniqueConstraint("path", name="uq_image_record_path"),
sa.UniqueConstraint("sha256", name="uq_image_record_sha256"),
)
op.create_index("ix_image_record_sha256", "image_record", ["sha256"])
op.create_index("ix_image_record_phash", "image_record", ["phash"])
op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"])
op.create_table(
"image_provenance",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("image_record_id", sa.Integer(), nullable=False),
sa.Column("post_id", sa.Integer(), nullable=False),
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("captured_metadata", sa.JSON(), nullable=True),
sa.Column(
"captured_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["image_record_id"],
["image_record.id"],
name="fk_image_provenance_image_record_id_image_record",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["post_id"],
["post.id"],
name="fk_image_provenance_post_id_post",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["source_id"],
["source.id"],
name="fk_image_provenance_source_id_source",
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name="pk_image_provenance"),
)
op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"])
op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"])
op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"])
op.create_table(
"tag",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("namespace", sa.String(length=64), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.PrimaryKeyConstraint("id", name="pk_tag"),
sa.UniqueConstraint("name", name="uq_tag_name"),
)
op.create_index("ix_tag_name", "tag", ["name"])
op.create_index("ix_tag_namespace", "tag", ["namespace"])
op.create_table(
"image_tag",
sa.Column("image_record_id", sa.Integer(), nullable=False),
sa.Column("tag_id", sa.Integer(), nullable=False),
sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["image_record_id"],
["image_record.id"],
name="fk_image_tag_image_record_id_image_record",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"),
)
op.create_table(
"download_event",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("source_id", sa.Integer(), nullable=False),
sa.Column("post_id", sa.Integer(), nullable=True),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column(
"started_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("error", sa.Text(), nullable=True),
sa.ForeignKeyConstraint(
["source_id"],
["source.id"],
name="fk_download_event_source_id_source",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["post_id"],
["post.id"],
name="fk_download_event_post_id_post",
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name="pk_download_event"),
)
op.create_index("ix_download_event_source_id", "download_event", ["source_id"])
op.create_index("ix_download_event_post_id", "download_event", ["post_id"])
def downgrade() -> None:
op.drop_table("download_event")
op.drop_table("image_tag")
op.drop_table("tag")
op.drop_table("image_provenance")
op.drop_table("image_record")
op.execute("DROP TYPE IF EXISTS origin_enum")
op.drop_table("post")
op.drop_table("credential")
op.drop_table("source")
op.drop_table("artist")
op.execute("DROP EXTENSION IF EXISTS vector")
+20 -3
View File
@@ -1,7 +1,24 @@
"""All ORM models. Import this module to make every model visible to Alembic.""" """All ORM models. Import this module to make every model visible to Alembic."""
from .artist import Artist
from .base import Base from .base import Base
from .credential import Credential
from .download_event import DownloadEvent
from .image_provenance import ImageProvenance
from .image_record import ImageRecord
from .post import Post
from .source import Source
from .tag import Tag, image_tag
# Concrete models land in Task 6 and re-export here. __all__ = [
"Base",
__all__ = ["Base"] "Artist",
"Source",
"Credential",
"Post",
"ImageRecord",
"ImageProvenance",
"Tag",
"image_tag",
"DownloadEvent",
]
+33
View File
@@ -0,0 +1,33 @@
"""Artist — unified entity that is both the gallery's ``artist:`` tag concept
and GallerySubscriber's Subscription. ``is_subscription`` is True if any
Sources are attached.
"""
from datetime import datetime
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from .base import Base
class Artist(Base):
__tablename__ = "artist"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
# True once a Source is attached; flips false if all sources removed.
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Per-artist scheduling overrides; null means "use global default".
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
sources = relationship("Source", back_populates="artist", cascade="all, delete-orphan")
+24
View File
@@ -0,0 +1,24 @@
"""Credential — encrypted blob (cookies/token/oauth) scoped per-platform,
not per-source. One Patreon credential serves every Patreon source.
"""
from datetime import datetime
from sqlalchemy import DateTime, Integer, LargeBinary, String, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class Credential(Base):
__tablename__ = "credential"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
platform: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
kind: Mapped[str] = mapped_column(String(32), nullable=False) # cookies|token|oauth
encrypted_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
captured_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+28
View File
@@ -0,0 +1,28 @@
"""DownloadEvent — log of every gallery-dl run, populated by FC-3."""
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class DownloadEvent(Base):
__tablename__ = "download_event"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_id: Mapped[int] = mapped_column(
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
)
post_id: Mapped[int | None] = mapped_column(
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
)
status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error
started_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
+33
View File
@@ -0,0 +1,33 @@
"""ImageProvenance — links an ImageRecord to a Post.
Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate
rule (spec §3): when a downloaded image is a pHash dupe of an existing
record, we append a new provenance row to the existing record rather than
dropping the metadata.
"""
from datetime import datetime
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class ImageProvenance(Base):
__tablename__ = "image_provenance"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
image_record_id: Mapped[int] = mapped_column(
ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True
)
post_id: Mapped[int] = mapped_column(
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
)
source_id: Mapped[int] = mapped_column(
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
)
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
captured_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
+71
View File
@@ -0,0 +1,71 @@
"""ImageRecord — the gallery's primary entity, ported from ImageRepo.
ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them
without a schema migration. The SigLIP embedding column uses pgvector's Vector
type — pgvector extension is enabled in the initial migration.
"""
from datetime import datetime
from pgvector.sqlalchemy import Vector
from sqlalchemy import (
JSON,
BigInteger,
DateTime,
Enum,
ForeignKey,
Integer,
String,
Text,
func,
)
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
class ImageRecord(Base):
__tablename__ = "image_record"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
# On-disk identity
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
mime: Mapped[str] = mapped_column(String(64), nullable=False)
width: Mapped[int | None] = mapped_column(Integer, nullable=True)
height: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Thumbnail (populated by FC-2)
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
# Origin / provenance pointers
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
primary_post_id: Mapped[int | None] = mapped_column(
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
)
# ML fields (populated by FC-2's ml-worker)
wd14_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True)
wd14_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
# 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
# a column-width migration.
siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True)
siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Centroid score cache (populated post-tagging)
centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
onupdate=func.now(),
)
+33
View File
@@ -0,0 +1,33 @@
"""Post — provenance anchor for content downloaded from a Source.
A Post is one creator post; it may contain many images/videos.
"""
from datetime import datetime
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class Post(Base):
__tablename__ = "post"
__table_args__ = (
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_id: Mapped[int] = mapped_column(
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
)
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
post_title: Mapped[str | None] = mapped_column(Text, nullable=True)
post_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
raw_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
downloaded_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
+31
View File
@@ -0,0 +1,31 @@
"""Source — a platform-specific URL owned by an Artist (e.g., a Patreon URL).
Multiple sources per artist support creators with cross-platform presence.
"""
from datetime import datetime
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column, relationship
from .base import Base
class Source(Base):
__tablename__ = "source"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
artist_id: Mapped[int] = mapped_column(
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
)
platform: Mapped[str] = mapped_column(String(64), nullable=False)
url: Mapped[str] = mapped_column(Text, nullable=False)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
artist = relationship("Artist", back_populates="sources")
+40
View File
@@ -0,0 +1,40 @@
"""Tag + ImageTag association.
Tags use namespaced strings (``artist:Name``, ``character:Name``,
``rating:safe``, etc.). Artist tags are kept in sync with the Artist table —
the sync trigger lands in FC-2.
"""
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Table, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from .base import Base
image_tag = Table(
"image_tag",
Base.metadata,
Column(
"image_record_id",
ForeignKey("image_record.id", ondelete="CASCADE"),
primary_key=True,
),
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
Column("source", String(32), nullable=False, default="manual"), # manual|auto|ml
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
)
class Tag(Base):
__tablename__ = "tag"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
namespace: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
images = relationship("ImageRecord", secondary=image_tag, backref="tags")
+29
View File
@@ -0,0 +1,29 @@
"""Smoke test — every model imports cleanly and shares the same metadata."""
from backend.app.models import (
Artist,
Base,
ImageRecord,
)
def test_all_models_use_shared_metadata():
expected_tables = {
"artist",
"source",
"credential",
"post",
"image_record",
"image_provenance",
"tag",
"image_tag",
"download_event",
}
assert expected_tables.issubset(Base.metadata.tables.keys())
def test_artist_has_subscription_flag():
# Sanity: the unified data model decision (Subscription IS Artist) shows up
# as a boolean flag on Artist, per spec §3.
assert "is_subscription" in {c.name for c in Artist.__table__.columns}
assert "primary_post_id" in {c.name for c in ImageRecord.__table__.columns}