feat: define unified schema (Artist, Source, Post, ImageRecord, etc.) and initial migration
Implements the data model from spec §3 in one go so FC-2/FC-3 don't need schema-adding migrations of their own. Artist is the unified entity for both gallery 'artist:' tags and GallerySubscriber Subscriptions (is_subscription flag). ImageProvenance is many-to-one, enabling the enrich-on-duplicate rule for downloaded content that pHash-matches an existing record. The SigLIP embedding column uses pgvector(1152) for SigLIP-so400m; swapping models in FC-2 will require a column-width migration. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,277 @@
|
|||||||
|
"""initial unified schema
|
||||||
|
|
||||||
|
Revision ID: 0001
|
||||||
|
Revises:
|
||||||
|
Create Date: 2026-05-13
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from pgvector.sqlalchemy import Vector
|
||||||
|
|
||||||
|
revision: str = "0001"
|
||||||
|
down_revision: Union[str, None] = None
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"artist",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("slug", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||||
|
sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
|
sa.Column("check_interval_seconds", sa.Integer(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_artist"),
|
||||||
|
sa.UniqueConstraint("name", name="uq_artist_name"),
|
||||||
|
sa.UniqueConstraint("slug", name="uq_artist_slug"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"source",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("artist_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("url", sa.Text(), nullable=False),
|
||||||
|
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
|
sa.Column("config_overrides", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("check_interval_override", sa.Integer(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE"
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_source"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_source_artist_id", "source", ["artist_id"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"credential",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=32), nullable=False, server_default="active"),
|
||||||
|
sa.Column(
|
||||||
|
"captured_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_credential"),
|
||||||
|
sa.UniqueConstraint("platform", name="uq_credential_platform"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"post",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("external_post_id", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("post_url", sa.Text(), nullable=True),
|
||||||
|
sa.Column("post_title", sa.Text(), nullable=True),
|
||||||
|
sa.Column("post_date", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("raw_metadata", sa.JSON(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"downloaded_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE"
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_post"),
|
||||||
|
sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_post_source_id", "post", ["source_id"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"image_record",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("path", sa.Text(), nullable=False),
|
||||||
|
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("phash", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column("mime", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("width", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("height", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("thumbnail_path", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"origin",
|
||||||
|
sa.Enum(
|
||||||
|
"downloaded",
|
||||||
|
"imported_filesystem",
|
||||||
|
"uploaded",
|
||||||
|
name="origin_enum",
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("primary_post_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("wd14_predictions", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("wd14_model_version", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("siglip_embedding", Vector(1152), nullable=True),
|
||||||
|
sa.Column("siglip_model_version", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("centroid_scores", sa.JSON(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["primary_post_id"],
|
||||||
|
["post.id"],
|
||||||
|
name="fk_image_record_primary_post_id_post",
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_image_record"),
|
||||||
|
sa.UniqueConstraint("path", name="uq_image_record_path"),
|
||||||
|
sa.UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_image_record_sha256", "image_record", ["sha256"])
|
||||||
|
op.create_index("ix_image_record_phash", "image_record", ["phash"])
|
||||||
|
op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"image_provenance",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("post_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("captured_metadata", sa.JSON(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"captured_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["image_record_id"],
|
||||||
|
["image_record.id"],
|
||||||
|
name="fk_image_provenance_image_record_id_image_record",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["post_id"],
|
||||||
|
["post.id"],
|
||||||
|
name="fk_image_provenance_post_id_post",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"],
|
||||||
|
["source.id"],
|
||||||
|
name="fk_image_provenance_source_id_source",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_image_provenance"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"])
|
||||||
|
op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"])
|
||||||
|
op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"tag",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("namespace", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_tag"),
|
||||||
|
sa.UniqueConstraint("name", name="uq_tag_name"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_tag_name", "tag", ["name"])
|
||||||
|
op.create_index("ix_tag_namespace", "tag", ["namespace"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"image_tag",
|
||||||
|
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["image_record_id"],
|
||||||
|
["image_record.id"],
|
||||||
|
name="fk_image_tag_image_record_id_image_record",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE"
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"download_event",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("post_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"started_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"],
|
||||||
|
["source.id"],
|
||||||
|
name="fk_download_event_source_id_source",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["post_id"],
|
||||||
|
["post.id"],
|
||||||
|
name="fk_download_event_post_id_post",
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_download_event"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_download_event_source_id", "download_event", ["source_id"])
|
||||||
|
op.create_index("ix_download_event_post_id", "download_event", ["post_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("download_event")
|
||||||
|
op.drop_table("image_tag")
|
||||||
|
op.drop_table("tag")
|
||||||
|
op.drop_table("image_provenance")
|
||||||
|
op.drop_table("image_record")
|
||||||
|
op.execute("DROP TYPE IF EXISTS origin_enum")
|
||||||
|
op.drop_table("post")
|
||||||
|
op.drop_table("credential")
|
||||||
|
op.drop_table("source")
|
||||||
|
op.drop_table("artist")
|
||||||
|
op.execute("DROP EXTENSION IF EXISTS vector")
|
||||||
@@ -1,7 +1,24 @@
|
|||||||
"""All ORM models. Import this module to make every model visible to Alembic."""
|
"""All ORM models. Import this module to make every model visible to Alembic."""
|
||||||
|
|
||||||
|
from .artist import Artist
|
||||||
from .base import Base
|
from .base import Base
|
||||||
|
from .credential import Credential
|
||||||
|
from .download_event import DownloadEvent
|
||||||
|
from .image_provenance import ImageProvenance
|
||||||
|
from .image_record import ImageRecord
|
||||||
|
from .post import Post
|
||||||
|
from .source import Source
|
||||||
|
from .tag import Tag, image_tag
|
||||||
|
|
||||||
# Concrete models land in Task 6 and re-export here.
|
__all__ = [
|
||||||
|
"Base",
|
||||||
__all__ = ["Base"]
|
"Artist",
|
||||||
|
"Source",
|
||||||
|
"Credential",
|
||||||
|
"Post",
|
||||||
|
"ImageRecord",
|
||||||
|
"ImageProvenance",
|
||||||
|
"Tag",
|
||||||
|
"image_tag",
|
||||||
|
"DownloadEvent",
|
||||||
|
]
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Artist — unified entity that is both the gallery's ``artist:`` tag concept
|
||||||
|
and GallerySubscriber's Subscription. ``is_subscription`` is True if any
|
||||||
|
Sources are attached.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Artist(Base):
|
||||||
|
__tablename__ = "artist"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
|
# True once a Source is attached; flips false if all sources removed.
|
||||||
|
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
|
||||||
|
# Per-artist scheduling overrides; null means "use global default".
|
||||||
|
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
sources = relationship("Source", back_populates="artist", cascade="all, delete-orphan")
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""Credential — encrypted blob (cookies/token/oauth) scoped per-platform,
|
||||||
|
not per-source. One Patreon credential serves every Patreon source.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Integer, LargeBinary, String, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Credential(Base):
|
||||||
|
__tablename__ = "credential"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
platform: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(32), nullable=False) # cookies|token|oauth
|
||||||
|
encrypted_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
|
||||||
|
captured_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""DownloadEvent — log of every gallery-dl run, populated by FC-3."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadEvent(Base):
|
||||||
|
__tablename__ = "download_event"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
post_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||||
|
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""ImageProvenance — links an ImageRecord to a Post.
|
||||||
|
|
||||||
|
Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate
|
||||||
|
rule (spec §3): when a downloaded image is a pHash dupe of an existing
|
||||||
|
record, we append a new provenance row to the existing record rather than
|
||||||
|
dropping the metadata.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ImageProvenance(Base):
|
||||||
|
__tablename__ = "image_provenance"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
image_record_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
post_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
captured_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""ImageRecord — the gallery's primary entity, ported from ImageRepo.
|
||||||
|
|
||||||
|
ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them
|
||||||
|
without a schema migration. The SigLIP embedding column uses pgvector's Vector
|
||||||
|
type — pgvector extension is enabled in the initial migration.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from pgvector.sqlalchemy import Vector
|
||||||
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
BigInteger,
|
||||||
|
DateTime,
|
||||||
|
Enum,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
func,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
|
||||||
|
|
||||||
|
|
||||||
|
class ImageRecord(Base):
|
||||||
|
__tablename__ = "image_record"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
|
||||||
|
# On-disk identity
|
||||||
|
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||||
|
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
||||||
|
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||||
|
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
width: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
height: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
|
# Thumbnail (populated by FC-2)
|
||||||
|
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
|
# Origin / provenance pointers
|
||||||
|
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
|
||||||
|
primary_post_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# ML fields (populated by FC-2's ml-worker)
|
||||||
|
wd14_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
wd14_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
# 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
|
||||||
|
# a column-width migration.
|
||||||
|
siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True)
|
||||||
|
siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
|
||||||
|
# Centroid score cache (populated post-tagging)
|
||||||
|
centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=func.now(),
|
||||||
|
onupdate=func.now(),
|
||||||
|
)
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Post — provenance anchor for content downloaded from a Source.
|
||||||
|
|
||||||
|
A Post is one creator post; it may contain many images/videos.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Post(Base):
|
||||||
|
__tablename__ = "post"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
post_title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
post_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
|
||||||
|
raw_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
|
||||||
|
downloaded_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""Source — a platform-specific URL owned by an Artist (e.g., a Patreon URL).
|
||||||
|
|
||||||
|
Multiple sources per artist support creators with cross-platform presence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Source(Base):
|
||||||
|
__tablename__ = "source"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
artist_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
|
||||||
|
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
|
||||||
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
|
artist = relationship("Artist", back_populates="sources")
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""Tag + ImageTag association.
|
||||||
|
|
||||||
|
Tags use namespaced strings (``artist:Name``, ``character:Name``,
|
||||||
|
``rating:safe``, etc.). Artist tags are kept in sync with the Artist table —
|
||||||
|
the sync trigger lands in FC-2.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Table, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
image_tag = Table(
|
||||||
|
"image_tag",
|
||||||
|
Base.metadata,
|
||||||
|
Column(
|
||||||
|
"image_record_id",
|
||||||
|
ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
),
|
||||||
|
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
|
||||||
|
Column("source", String(32), nullable=False, default="manual"), # manual|auto|ml
|
||||||
|
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Tag(Base):
|
||||||
|
__tablename__ = "tag"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
|
||||||
|
namespace: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||||
|
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
images = relationship("ImageRecord", secondary=image_tag, backref="tags")
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""Smoke test — every model imports cleanly and shares the same metadata."""
|
||||||
|
|
||||||
|
from backend.app.models import (
|
||||||
|
Artist,
|
||||||
|
Base,
|
||||||
|
ImageRecord,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_models_use_shared_metadata():
|
||||||
|
expected_tables = {
|
||||||
|
"artist",
|
||||||
|
"source",
|
||||||
|
"credential",
|
||||||
|
"post",
|
||||||
|
"image_record",
|
||||||
|
"image_provenance",
|
||||||
|
"tag",
|
||||||
|
"image_tag",
|
||||||
|
"download_event",
|
||||||
|
}
|
||||||
|
assert expected_tables.issubset(Base.metadata.tables.keys())
|
||||||
|
|
||||||
|
|
||||||
|
def test_artist_has_subscription_flag():
|
||||||
|
# Sanity: the unified data model decision (Subscription IS Artist) shows up
|
||||||
|
# as a boolean flag on Artist, per spec §3.
|
||||||
|
assert "is_subscription" in {c.name for c in Artist.__table__.columns}
|
||||||
|
assert "primary_post_id" in {c.name for c in ImageRecord.__table__.columns}
|
||||||
Reference in New Issue
Block a user