bvandeusen
3610ba495f
Read cutover verified in prod (suggestions + allowlist read image_prediction; backfill complete at 908k rows / 51k images). Removes the old JSON column and everything that fed it: - ImageRecord.tagger_predictions column removed; migration 0046 DROPs it. tagger_model_version kept as the "tagged / current?" signal the backfill sweep reads (needs-tagging check switched to tagger_model_version IS NULL). - tag_and_embed no longer dual-writes the JSON — image_prediction is the only write path. - importer re-import reset drops the JSON line (image_prediction rows are already deleted on re-import). - Retired the one-time #768 backfill task + the #764 prune task, their admin endpoints, and their Maintenance cards (Backfill/PrunePredictionsCard). - Tests seed/assert via image_prediction; stale column refs removed. Disk reclaim is NOT automatic: DROP COLUMN is a catalog change. Run `VACUUM FULL image_record` off-hours afterward to return the ~100 GB to the OS so DB backups go small (#739). image_prediction (~90 MB) stays in pg_dump — it's the source of truth now. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
96 lines
4.1 KiB
Python
96 lines
4.1 KiB
Python
"""ImageRecord — the gallery's primary entity, ported from ImageRepo.
|
||
|
||
ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them
|
||
without a schema migration. The SigLIP embedding column uses pgvector's Vector
|
||
type — pgvector extension is enabled in the initial migration.
|
||
"""
|
||
|
||
from datetime import datetime
|
||
|
||
from pgvector.sqlalchemy import Vector
|
||
from sqlalchemy import (
|
||
JSON,
|
||
BigInteger,
|
||
DateTime,
|
||
Enum,
|
||
ForeignKey,
|
||
Integer,
|
||
String,
|
||
Text,
|
||
func,
|
||
)
|
||
from sqlalchemy.orm import Mapped, mapped_column
|
||
|
||
from .base import Base
|
||
|
||
ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
|
||
|
||
|
||
class ImageRecord(Base):
|
||
__tablename__ = "image_record"
|
||
|
||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||
|
||
# On-disk identity
|
||
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
||
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
width: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||
height: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||
|
||
# Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
|
||
# Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
|
||
integrity_status: Mapped[str] = mapped_column(
|
||
String(24), nullable=False, default="unknown", index=True
|
||
)
|
||
|
||
# Thumbnail (populated by FC-2)
|
||
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||
|
||
# Origin / provenance pointers
|
||
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
|
||
primary_post_id: Mapped[int | None] = mapped_column(
|
||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||
)
|
||
# FC-2d-vii-c: canonical per-image artist (the single source of truth
|
||
# for attribution; provenance posts remain lineage detail).
|
||
artist_id: Mapped[int | None] = mapped_column(
|
||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||
)
|
||
|
||
# ML fields (populated by FC-2's ml-worker). Per-tag predictions live in the
|
||
# normalized image_prediction table (#768) — the tagger_predictions JSON
|
||
# column was dropped in migration 0046. tagger_model_version stays as the
|
||
# "has this been tagged / is it current?" signal the backfill sweep reads.
|
||
tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||
# 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
|
||
# a column-width migration.
|
||
siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True)
|
||
siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||
|
||
# Centroid score cache (populated post-tagging)
|
||
centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||
|
||
created_at: Mapped[datetime] = mapped_column(
|
||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||
)
|
||
# Denormalized gallery sort key = COALESCE(primary post's post_date,
|
||
# created_at) (alembic 0035). The gallery used to compute this as a
|
||
# COALESCE across the Post outer join on every /scroll, which can't use
|
||
# an index and re-sorted a large slice of the library per page (×10 with
|
||
# the old serial batching). Materializing it lets the cursor scroll read
|
||
# ix_image_record_effective_date directly. Maintained by the importer
|
||
# (services/importer.py _apply_sidecar) when a primary post with a date
|
||
# is linked; plain inserts keep the created_at-equivalent server default.
|
||
effective_date: Mapped[datetime] = mapped_column(
|
||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||
)
|
||
updated_at: Mapped[datetime] = mapped_column(
|
||
DateTime(timezone=True),
|
||
nullable=False,
|
||
server_default=func.now(),
|
||
onupdate=func.now(),
|
||
)
|