diff --git a/alembic/versions/0035_image_record_effective_date.py b/alembic/versions/0035_image_record_effective_date.py new file mode 100644 index 0000000..586cf51 --- /dev/null +++ b/alembic/versions/0035_image_record_effective_date.py @@ -0,0 +1,70 @@ +"""image_record.effective_date: materialized gallery sort key + index + +Revision ID: 0035 +Revises: 0034 +Create Date: 2026-06-04 + +The gallery ordered/cursored on COALESCE(post.post_date, +image_record.created_at) across the Post outer join. That expression spans +two tables, so no index can serve it — every /scroll sorted a large slice +of the library, and the frontend fired ten of them serially per initial +load. Materialize the value into image_record.effective_date and index +(effective_date DESC, id DESC) so the cursor scroll is an index range scan. + +Backfill = COALESCE(primary post's post_date, created_at) so existing rows +keep their exact ordering. New rows get the created_at-equivalent server +default; services/importer.py overrides it with the post's date when a +primary post with a date is linked. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0035" +down_revision: Union[str, None] = "0034" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable first so the backfill can populate before NOT NULL. + op.add_column( + "image_record", + sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True), + ) + # Pure set-based UPDATEs (no per-row params) — immune to the 65535 + # bind-parameter ceiling regardless of library size. + op.execute( + """ + UPDATE image_record AS ir + SET effective_date = COALESCE(p.post_date, ir.created_at) + FROM post AS p + WHERE ir.primary_post_id = p.id + """ + ) + op.execute( + """ + UPDATE image_record + SET effective_date = created_at + WHERE effective_date IS NULL + """ + ) + op.alter_column( + "image_record", + "effective_date", + nullable=False, + server_default=sa.text("now()"), + ) + # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC + # so the scroll is a forward index scan; raw SQL because alembic's + # column list doesn't express per-column DESC cleanly. + op.execute( + "CREATE INDEX ix_image_record_effective_date " + "ON image_record (effective_date DESC, id DESC)" + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_effective_date", table_name="image_record") + op.drop_column("image_record", "effective_date") diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index f43fd60..812a9fa 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -74,6 +74,17 @@ class ImageRecord(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + # Denormalized gallery sort key = COALESCE(primary post's post_date, + # created_at) (alembic 0035). The gallery used to compute this as a + # COALESCE across the Post outer join on every /scroll, which can't use + # an index and re-sorted a large slice of the library per page (×10 with + # the old serial batching). Materializing it lets the cursor scroll read + # ix_image_record_effective_date directly. Maintained by the importer + # (services/importer.py _apply_sidecar) when a primary post with a date + # is linked; plain inserts keep the created_at-equivalent server default. + effective_date: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 3331510..d0eb09b 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -43,16 +43,17 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]: def _effective_date_col(): - """SQL expression: COALESCE(post.post_date, image_record.created_at). + """The materialized gallery sort key: image_record.effective_date + (alembic 0035) = COALESCE(primary post's post_date, created_at), + maintained at write time by the importer. - Used as the canonical sort/group/filter key across the gallery so - images backfilled with primary_post_id (e.g. via tag_apply phase 4) - surface at their original publish date, not their FC import date. - Images without a Post (or with Post.post_date NULL) fall back to - image_record.created_at and still order coherently against - post-attached ones. + Canonical sort/group/filter key across the gallery so images attached + to a post surface at their original publish date, not their FC import + date — and, now that it's a single indexed column rather than a + COALESCE across the Post outer join, the cursor scroll is an index + range scan instead of a full re-sort per page. """ - return func.coalesce(Post.post_date, ImageRecord.created_at) + return ImageRecord.effective_date def _outer_join_primary_post(stmt: Select) -> Select: @@ -357,17 +358,10 @@ class GalleryService: } async def _neighbors(self, record: ImageRecord) -> dict: - # Compute the boundary image's effective_date in Python (one query - # below + the SELECT we already have on `record`) and use it for - # the neighbor comparison. Cheaper than re-deriving in SQL via - # correlated subquery. - boundary_eff = record.created_at - if record.primary_post_id is not None: - post_date = (await self.session.execute( - select(Post.post_date).where(Post.id == record.primary_post_id) - )).scalar_one_or_none() - if post_date is not None: - boundary_eff = post_date + # The boundary image's sort key is materialized on the row now + # (alembic 0035) — read it directly instead of re-deriving COALESCE + # via an extra Post lookup. + boundary_eff = record.effective_date eff = _effective_date_col() prev_stmt = _outer_join_primary_post( diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 72713c5..213b86e 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -960,6 +960,14 @@ class Importer: sp.rollback() if record.primary_post_id is None: record.primary_post_id = post.id + # Keep the denormalized gallery sort key (alembic 0035) aligned with + # the primary post's publish date so /scroll orders off + # ix_image_record_effective_date instead of COALESCE-ing across the + # post join. Only override when THIS post is the primary AND carries + # a date; otherwise the column keeps its created_at-equivalent server + # default (matches the old COALESCE(post_date, created_at) fallback). + if record.primary_post_id == post.id and post.post_date is not None: + record.effective_date = post.post_date self.session.flush() def _copy_to_library( diff --git a/tests/test_api_gallery.py b/tests/test_api_gallery.py index f64066e..b15a690 100644 --- a/tests/test_api_gallery.py +++ b/tests/test_api_gallery.py @@ -17,6 +17,7 @@ async def _seed(db, count: int = 3): origin="imported_filesystem", integrity_status="unknown", ) r.created_at = base - timedelta(minutes=i) + r.effective_date = r.created_at # no post → tracks created_at (0035) db.add(r) await db.flush() await db.commit() diff --git a/tests/test_gallery_artist_payload.py b/tests/test_gallery_artist_payload.py index 46c7099..9f15ce1 100644 --- a/tests/test_gallery_artist_payload.py +++ b/tests/test_gallery_artist_payload.py @@ -14,6 +14,7 @@ async def _img(db, n): origin="imported_filesystem", integrity_status="unknown", ) rec.created_at = datetime.now(UTC) - timedelta(minutes=n) + rec.effective_date = rec.created_at # no post → tracks created_at (0035) db.add(rec) await db.flush() return rec diff --git a/tests/test_gallery_provenance_filter.py b/tests/test_gallery_provenance_filter.py index 10b2ffa..1cda34a 100644 --- a/tests/test_gallery_provenance_filter.py +++ b/tests/test_gallery_provenance_filter.py @@ -22,6 +22,7 @@ async def _img(db, n): origin="imported_filesystem", integrity_status="unknown", ) rec.created_at = base - timedelta(minutes=n) + rec.effective_date = rec.created_at # no primary post → tracks created_at (0035) db.add(rec) await db.flush() return rec diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index 89d0c53..f1e553d 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -32,6 +32,8 @@ async def _seed_images(db, count: int, sha_prefix: str = "0") -> list[ImageRecor integrity_status="unknown", ) r.created_at = base - timedelta(minutes=i) + # No post → effective_date == created_at (alembic 0035 denorm). + r.effective_date = r.created_at db.add(r) records.append(r) await db.flush() @@ -166,6 +168,9 @@ async def _seed_image_with_post( primary_post_id=post.id, ) img.created_at = image_created_at + # Mirror the importer's denorm (alembic 0035): a linked post with a date + # sets effective_date to that date, else it falls back to created_at. + img.effective_date = post_date or image_created_at db.add(img) await db.flush() return img, post @@ -201,6 +206,7 @@ async def test_scroll_sorts_by_post_date_when_available(db): origin="imported_filesystem", integrity_status="unknown", ) img_c.created_at = base_import - timedelta(days=5) + img_c.effective_date = img_c.created_at # no post → tracks created_at db.add(img_c) await db.flush() diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index 1984d99..0457238 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -94,6 +94,9 @@ def test_sidecar_creates_provenance(importer, import_layout): prov = importer.session.execute(select(ImageProvenance)).scalar_one() assert prov.image_record_id == rec.id and prov.post_id == post.id assert rec.primary_post_id == post.id + # Denormalized gallery sort key (alembic 0035) tracks the primary post's + # publish date so /scroll orders off ix_image_record_effective_date. + assert rec.effective_date == post.post_date def test_reimport_same_post_idempotent(importer, import_layout):