perf(gallery): materialize indexed effective_date sort key

The gallery cursored on COALESCE(post.post_date, image_record.created_at)
across the Post outer join — an expression spanning two tables that no
index can serve, so every /scroll sorted a large slice of the library
(and the old frontend fired ten serially). Materialize it:

- image_record.effective_date column + ix_image_record_effective_date
  (effective_date DESC, id DESC); alembic 0035 backfills
  COALESCE(primary post's post_date, created_at) for existing rows.
- gallery_service._effective_date_col() now returns the column, so scroll
  / timeline / jump / neighbors all order off the index instead of
  re-deriving the COALESCE. _neighbors reads record.effective_date
  directly (drops an extra Post lookup).
- importer._apply_sidecar maintains it: when a primary post with a date is
  linked, effective_date = post.post_date; plain inserts keep the
  created_at-equivalent server default.

Tests: sidecar import asserts effective_date == post.post_date; gallery
ordering/timeline/jump test seeds set effective_date alongside created_at.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

alembic/versions/0035_image_record_effective_date.py

@@ -0,0 +1,70 @@
+"""image_record.effective_date: materialized gallery sort key + index
+
+Revision ID: 0035
+Revises: 0034
+Create Date: 2026-06-04
+
+The gallery ordered/cursored on COALESCE(post.post_date,
+image_record.created_at) across the Post outer join. That expression spans
+two tables, so no index can serve it — every /scroll sorted a large slice
+of the library, and the frontend fired ten of them serially per initial
+load. Materialize the value into image_record.effective_date and index
+(effective_date DESC, id DESC) so the cursor scroll is an index range scan.
+
+Backfill = COALESCE(primary post's post_date, created_at) so existing rows
+keep their exact ordering. New rows get the created_at-equivalent server
+default; services/importer.py overrides it with the post's date when a
+primary post with a date is linked.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0035"
+down_revision: Union[str, None] = "0034"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add nullable first so the backfill can populate before NOT NULL.
+    op.add_column(
+        "image_record",
+        sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True),
+    )
+    # Pure set-based UPDATEs (no per-row params) — immune to the 65535
+    # bind-parameter ceiling regardless of library size.
+    op.execute(
+        """
+        UPDATE image_record AS ir
+        SET effective_date = COALESCE(p.post_date, ir.created_at)
+        FROM post AS p
+        WHERE ir.primary_post_id = p.id
+        """
+    )
+    op.execute(
+        """
+        UPDATE image_record
+        SET effective_date = created_at
+        WHERE effective_date IS NULL
+        """
+    )
+    op.alter_column(
+        "image_record",
+        "effective_date",
+        nullable=False,
+        server_default=sa.text("now()"),
+    )
+    # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC
+    # so the scroll is a forward index scan; raw SQL because alembic's
+    # column list doesn't express per-column DESC cleanly.
+    op.execute(
+        "CREATE INDEX ix_image_record_effective_date "
+        "ON image_record (effective_date DESC, id DESC)"
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_image_record_effective_date", table_name="image_record")
+    op.drop_column("image_record", "effective_date")

backend/app/models/image_record.py

@@ -74,6 +74,17 @@ class ImageRecord(Base):
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
+    # Denormalized gallery sort key = COALESCE(primary post's post_date,
+    # created_at) (alembic 0035). The gallery used to compute this as a
+    # COALESCE across the Post outer join on every /scroll, which can't use
+    # an index and re-sorted a large slice of the library per page (×10 with
+    # the old serial batching). Materializing it lets the cursor scroll read
+    # ix_image_record_effective_date directly. Maintained by the importer
+    # (services/importer.py _apply_sidecar) when a primary post with a date
+    # is linked; plain inserts keep the created_at-equivalent server default.
+    effective_date: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,

backend/app/services/gallery_service.py

@@ -43,16 +43,17 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]:
 
 
 def _effective_date_col():
-    """SQL expression: COALESCE(post.post_date, image_record.created_at).
+    """The materialized gallery sort key: image_record.effective_date
+    (alembic 0035) = COALESCE(primary post's post_date, created_at),
+    maintained at write time by the importer.
 
-    Used as the canonical sort/group/filter key across the gallery so
+    Canonical sort/group/filter key across the gallery so images attached
-    images backfilled with primary_post_id (e.g. via tag_apply phase 4)
+    to a post surface at their original publish date, not their FC import
-    surface at their original publish date, not their FC import date.
+    date — and, now that it's a single indexed column rather than a
-    Images without a Post (or with Post.post_date NULL) fall back to
+    COALESCE across the Post outer join, the cursor scroll is an index
-    image_record.created_at and still order coherently against
+    range scan instead of a full re-sort per page.
-    post-attached ones.
     """
-    return func.coalesce(Post.post_date, ImageRecord.created_at)
+    return ImageRecord.effective_date
 
 
 def _outer_join_primary_post(stmt: Select) -> Select:
@@ -357,17 +358,10 @@ class GalleryService:
         }
 
     async def _neighbors(self, record: ImageRecord) -> dict:
-        # Compute the boundary image's effective_date in Python (one query
+        # The boundary image's sort key is materialized on the row now
-        # below + the SELECT we already have on `record`) and use it for
+        # (alembic 0035) — read it directly instead of re-deriving COALESCE
-        # the neighbor comparison. Cheaper than re-deriving in SQL via
+        # via an extra Post lookup.
-        # correlated subquery.
+        boundary_eff = record.effective_date
-        boundary_eff = record.created_at
-        if record.primary_post_id is not None:
-            post_date = (await self.session.execute(
-                select(Post.post_date).where(Post.id == record.primary_post_id)
-            )).scalar_one_or_none()
-            if post_date is not None:
-                boundary_eff = post_date
 
         eff = _effective_date_col()
         prev_stmt = _outer_join_primary_post(

backend/app/services/importer.py

@@ -960,6 +960,14 @@ class Importer:
                 sp.rollback()
         if record.primary_post_id is None:
             record.primary_post_id = post.id
+        # Keep the denormalized gallery sort key (alembic 0035) aligned with
+        # the primary post's publish date so /scroll orders off
+        # ix_image_record_effective_date instead of COALESCE-ing across the
+        # post join. Only override when THIS post is the primary AND carries
+        # a date; otherwise the column keeps its created_at-equivalent server
+        # default (matches the old COALESCE(post_date, created_at) fallback).
+        if record.primary_post_id == post.id and post.post_date is not None:
+            record.effective_date = post.post_date
         self.session.flush()
 
     def _copy_to_library(

tests/test_api_gallery.py

@@ -17,6 +17,7 @@ async def _seed(db, count: int = 3):
             origin="imported_filesystem", integrity_status="unknown",
         )
         r.created_at = base - timedelta(minutes=i)
+        r.effective_date = r.created_at  # no post → tracks created_at (0035)
         db.add(r)
     await db.flush()
     await db.commit()

tests/test_gallery_artist_payload.py

@@ -14,6 +14,7 @@ async def _img(db, n):
         origin="imported_filesystem", integrity_status="unknown",
     )
     rec.created_at = datetime.now(UTC) - timedelta(minutes=n)
+    rec.effective_date = rec.created_at  # no post → tracks created_at (0035)
     db.add(rec)
     await db.flush()
     return rec

tests/test_gallery_provenance_filter.py

@@ -22,6 +22,7 @@ async def _img(db, n):
         origin="imported_filesystem", integrity_status="unknown",
     )
     rec.created_at = base - timedelta(minutes=n)
+    rec.effective_date = rec.created_at  # no primary post → tracks created_at (0035)
     db.add(rec)
     await db.flush()
     return rec

tests/test_gallery_service.py

@@ -32,6 +32,8 @@ async def _seed_images(db, count: int, sha_prefix: str = "0") -> list[ImageRecor
             integrity_status="unknown",
         )
         r.created_at = base - timedelta(minutes=i)
+        # No post → effective_date == created_at (alembic 0035 denorm).
+        r.effective_date = r.created_at
         db.add(r)
         records.append(r)
     await db.flush()
@@ -166,6 +168,9 @@ async def _seed_image_with_post(
         primary_post_id=post.id,
     )
     img.created_at = image_created_at
+    # Mirror the importer's denorm (alembic 0035): a linked post with a date
+    # sets effective_date to that date, else it falls back to created_at.
+    img.effective_date = post_date or image_created_at
     db.add(img)
     await db.flush()
     return img, post
@@ -201,6 +206,7 @@ async def test_scroll_sorts_by_post_date_when_available(db):
         origin="imported_filesystem", integrity_status="unknown",
     )
     img_c.created_at = base_import - timedelta(days=5)
+    img_c.effective_date = img_c.created_at  # no post → tracks created_at
     db.add(img_c)
     await db.flush()
 

tests/test_sidecar_import.py

@@ -94,6 +94,9 @@ def test_sidecar_creates_provenance(importer, import_layout):
     prov = importer.session.execute(select(ImageProvenance)).scalar_one()
     assert prov.image_record_id == rec.id and prov.post_id == post.id
     assert rec.primary_post_id == post.id
+    # Denormalized gallery sort key (alembic 0035) tracks the primary post's
+    # publish date so /scroll orders off ix_image_record_effective_date.
+    assert rec.effective_date == post.post_date
 
 
 def test_reimport_same_post_idempotent(importer, import_layout):