feat(gallery): sort/group/jump by COALESCE(post.post_date, image_record.created_at) — surface migrated content at its original publish date, not FC scan date

Operator hit this 2026-05-25 after the IR tag_apply landed: ~57k images
all scanned into FC in the same week share image_record.created_at, so
the gallery timeline collapses them into a single month bucket and
scroll orders them all together at the top. Their actual publish dates
(spread over years) were already available in Post.post_date but the
gallery never read it.

Backend wire-up:
- tag_apply phase 4 now sets ImageRecord.primary_post_id when creating
  ImageProvenance (only if currently NULL — preserves the canonical
  download-time linkage set by the importer for new FC ingests).
- gallery_service.py introduces _effective_date_col() =
  COALESCE(post.post_date, image_record.created_at), used in:
    * scroll() ORDER BY + cursor WHERE clauses
    * timeline() year/month group-by
    * jump_cursor() year/month filter
    * _neighbors() prev/next ordering
- Each method LEFT OUTER JOIN Post on primary_post_id so the COALESCE
  works for images without a post (NULL on the Post side, fall back
  to created_at).
- GalleryImage gains posted_at + effective_date fields; API /gallery
  /scroll exposes both alongside the existing created_at so the UI
  can render 'Posted on X (imported Y)' if desired.
- get_image_with_tags() returns posted_at for the modal.

Cursor format unchanged — the encoded datetime is now the effective_
date (whichever column won the COALESCE) and pagination remains
consistent.

To pick up new behavior for an already-migrated IR set: re-run
/api/migrate/tag_apply on the existing manifest (phase 4 is
idempotent; the new primary_post_id assignment backfills).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-25 12:30:18 -04:00
parent 9f54efdedf
commit c361032554
5 changed files with 328 additions and 55 deletions
+2
View File
@@ -42,6 +42,8 @@ async def scroll():
"width": i.width,
"height": i.height,
"created_at": i.created_at.isoformat(),
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
"effective_date": i.effective_date.isoformat(),
"thumbnail_url": i.thumbnail_url,
"artist": i.artist,
}
+113 -54
View File
@@ -1,26 +1,34 @@
"""Cursor-paginated gallery queries.
Cursor format: opaque base64-encoded "<iso8601_created_at>:<image_id>".
Pagination key is (created_at DESC, id DESC) so we don't drift when new
imports arrive between page loads. Decoding rejects malformed cursors with
a ValueError; the API layer translates that to HTTP 400.
Cursor format: opaque base64-encoded "<iso8601_effective_date>:<image_id>".
Pagination key is (effective_date DESC, id DESC) where effective_date is
COALESCE(post.post_date, image_record.created_at) so the gallery surfaces
images by ORIGINAL publish date when known, falling back to FC's scan
date. Important for migrated content: ~57k IR images scanned in a single
week would otherwise all share the same created_at and pile up in one
month bucket. The effective_date spreads them across the years they
were originally published.
Decoding rejects malformed cursors with a ValueError; the API layer
translates that to HTTP 400.
"""
import base64
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import and_, exists, func, or_, select
from sqlalchemy import Select, and_, exists, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import Artist, ImageProvenance, ImageRecord, Source, Tag
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
from ..models.tag import image_tag
CURSOR_SEPARATOR = "|"
def encode_cursor(created_at: datetime, image_id: int) -> str:
raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}"
def encode_cursor(effective_date: datetime, image_id: int) -> str:
raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}"
return base64.urlsafe_b64encode(raw.encode()).decode()
@@ -33,6 +41,26 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]:
raise ValueError(f"invalid cursor: {cursor!r}") from exc
def _effective_date_col():
"""SQL expression: COALESCE(post.post_date, image_record.created_at).
Used as the canonical sort/group/filter key across the gallery so
images backfilled with primary_post_id (e.g. via tag_apply phase 4)
surface at their original publish date, not their FC import date.
Images without a Post (or with Post.post_date NULL) fall back to
image_record.created_at and still order coherently against
post-attached ones.
"""
return func.coalesce(Post.post_date, ImageRecord.created_at)
def _outer_join_primary_post(stmt: Select) -> Select:
"""LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
above sees Post.post_date when available. Images without a post
survive the join as NULL on the Post side; COALESCE handles it."""
return stmt.outerjoin(Post, Post.id == ImageRecord.primary_post_id)
@dataclass(frozen=True)
class GalleryImage:
id: int
@@ -41,7 +69,9 @@ class GalleryImage:
mime: str
width: int | None
height: int | None
created_at: datetime
created_at: datetime # FC's row-insert time
effective_date: datetime # COALESCE(post.post_date, created_at)
posted_at: datetime | None # post.post_date if known, else None
thumbnail_url: str
artist: dict | None = None
@@ -78,7 +108,7 @@ def _require_single_filter(tag_id, post_id, artist_id) -> None:
def _provenance_clause(post_id, artist_id):
"""Correlated EXISTS clause (NOT a join) so an image with multiple
matching provenance rows is returned exactly once and the
(created_at DESC, id DESC) cursor ordering is unaffected."""
(effective_date DESC, id DESC) cursor ordering is unaffected."""
if post_id is not None:
return exists().where(
ImageProvenance.image_record_id == ImageRecord.id,
@@ -125,7 +155,9 @@ class GalleryService:
raise ValueError("limit must be between 1 and 200")
_require_single_filter(tag_id, post_id, artist_id)
stmt = select(ImageRecord)
eff = _effective_date_col()
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
stmt = _outer_join_primary_post(stmt)
if tag_id is not None:
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
image_tag.c.tag_id == tag_id
@@ -138,34 +170,38 @@ class GalleryService:
cur_ts, cur_id = decode_cursor(cursor)
stmt = stmt.where(
or_(
ImageRecord.created_at < cur_ts,
and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id),
eff < cur_ts,
and_(eff == cur_ts, ImageRecord.id < cur_id),
)
)
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1)
rows = (await self.session.execute(stmt)).scalars().all()
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1)
rows = (await self.session.execute(stmt)).all()
next_cursor = None
if len(rows) > limit:
last = rows[limit - 1]
next_cursor = encode_cursor(last.created_at, last.id)
last_record, _last_posted_at, last_eff = rows[limit - 1]
next_cursor = encode_cursor(last_eff, last_record.id)
rows = rows[:limit]
artists = await _artists_for(self.session, [r.id for r in rows])
artists = await _artists_for(
self.session, [r[0].id for r in rows]
)
images = [
GalleryImage(
id=r.id,
path=r.path,
sha256=r.sha256,
mime=r.mime,
width=r.width,
height=r.height,
created_at=r.created_at,
thumbnail_url=thumbnail_url(r.sha256, r.mime),
artist=artists.get(r.id),
id=record.id,
path=record.path,
sha256=record.sha256,
mime=record.mime,
width=record.width,
height=record.height,
created_at=record.created_at,
effective_date=eff_date,
posted_at=posted_at,
thumbnail_url=thumbnail_url(record.sha256, record.mime),
artist=artists.get(record.id),
)
for r in rows
for record, posted_at, eff_date in rows
]
return GalleryPage(
images=images,
@@ -179,11 +215,13 @@ class GalleryService:
post_id: int | None = None,
artist_id: int | None = None,
) -> list[TimelineBucket]:
year_col = func.date_part("year", ImageRecord.created_at).label("yr")
month_col = func.date_part("month", ImageRecord.created_at).label("mo")
eff = _effective_date_col()
year_col = func.date_part("year", eff).label("yr")
month_col = func.date_part("month", eff).label("mo")
stmt = select(
year_col, month_col, func.count(ImageRecord.id).label("cnt")
)
stmt = _outer_join_primary_post(stmt)
_require_single_filter(tag_id, post_id, artist_id)
if tag_id is not None:
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
@@ -201,14 +239,17 @@ class GalleryService:
post_id: int | None = None, artist_id: int | None = None,
) -> str | None:
"""Returns a cursor that, when passed to scroll(), positions at the
first image of the given year-month. None if the bucket is empty.
first image of the given year-month (by effective_date, not
created_at). None if the bucket is empty.
"""
from sqlalchemy import extract
stmt = select(ImageRecord).where(
extract("year", ImageRecord.created_at) == year,
extract("month", ImageRecord.created_at) == month,
eff = _effective_date_col()
stmt = select(ImageRecord, eff.label("eff")).where(
extract("year", eff) == year,
extract("month", eff) == month,
)
stmt = _outer_join_primary_post(stmt)
_require_single_filter(tag_id, post_id, artist_id)
if tag_id is not None:
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
@@ -217,13 +258,14 @@ class GalleryService:
prov = _provenance_clause(post_id, artist_id)
if prov is not None:
stmt = stmt.where(prov)
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1)
first = (await self.session.execute(stmt)).scalar_one_or_none()
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
first = (await self.session.execute(stmt)).first()
if first is None:
return None
record, eff_date = first
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
# is the first result in the next scroll().
return encode_cursor(first.created_at, first.id + 1)
return encode_cursor(eff_date, record.id + 1)
async def get_image_with_tags(self, image_id: int) -> dict | None:
record = await self.session.get(ImageRecord, image_id)
@@ -236,6 +278,13 @@ class GalleryService:
.order_by(Tag.kind.asc(), Tag.name.asc())
)
tags = (await self.session.execute(tag_stmt)).scalars().all()
# Fetch the canonical post.post_date for this image (if any) so
# the modal can show "Posted on <date>" alongside import date.
posted_at = None
if record.primary_post_id is not None:
posted_at = (await self.session.execute(
select(Post.post_date).where(Post.id == record.primary_post_id)
)).scalar_one_or_none()
neighbors = await self._neighbors(record)
# Direct artist FK — used by the modal's ProvenancePanel as a
# fallback when ImageProvenance is empty (i.e., filesystem-
@@ -256,6 +305,7 @@ class GalleryService:
"size_bytes": record.size_bytes,
"integrity_status": record.integrity_status,
"created_at": record.created_at.isoformat(),
"posted_at": posted_at.isoformat() if posted_at else None,
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
"artist": (
@@ -275,34 +325,41 @@ class GalleryService:
}
async def _neighbors(self, record: ImageRecord) -> dict:
prev_stmt = (
select(ImageRecord.id)
.where(
# Compute the boundary image's effective_date in Python (one query
# below + the SELECT we already have on `record`) and use it for
# the neighbor comparison. Cheaper than re-deriving in SQL via
# correlated subquery.
boundary_eff = record.created_at
if record.primary_post_id is not None:
post_date = (await self.session.execute(
select(Post.post_date).where(Post.id == record.primary_post_id)
)).scalar_one_or_none()
if post_date is not None:
boundary_eff = post_date
eff = _effective_date_col()
prev_stmt = _outer_join_primary_post(
select(ImageRecord.id).where(
or_(
ImageRecord.created_at > record.created_at,
eff > boundary_eff,
and_(
ImageRecord.created_at == record.created_at,
eff == boundary_eff,
ImageRecord.id > record.id,
),
)
)
.order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc())
.limit(1)
)
next_stmt = (
select(ImageRecord.id)
.where(
).order_by(eff.asc(), ImageRecord.id.asc()).limit(1)
next_stmt = _outer_join_primary_post(
select(ImageRecord.id).where(
or_(
ImageRecord.created_at < record.created_at,
eff < boundary_eff,
and_(
ImageRecord.created_at == record.created_at,
eff == boundary_eff,
ImageRecord.id < record.id,
),
)
)
.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc())
.limit(1)
)
).order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
return {"prev_id": prev_id, "next_id": next_id}
@@ -311,9 +368,11 @@ class GalleryService:
def _group_by_year_month(
images: list[GalleryImage],
) -> list[tuple[int, int, list[int]]]:
"""Group by effective_date's year/month so migrated content surfaces
in the publish-date buckets, not the FC-scan-date bucket."""
groups: list[tuple[int, int, list[int]]] = []
for img in images:
y, m = img.created_at.year, img.created_at.month
y, m = img.effective_date.year, img.effective_date.month
if groups and groups[-1][0] == y and groups[-1][1] == m:
groups[-1][2].append(img.id)
else:
+20 -1
View File
@@ -122,7 +122,14 @@ async def _ensure_provenance(
db: AsyncSession, *,
image_id: int, post_id: int, source_id: int, dry_run: bool,
) -> bool:
"""Returns True if a new ImageProvenance row was inserted."""
"""Returns True if a new ImageProvenance row was inserted.
Also sets ImageRecord.primary_post_id to this post if the image
doesn't already have one — preserves any primary_post_id already
assigned at download time by the importer (don't clobber). This is
the linkage gallery_service.py uses to surface Post.post_date as
the image's effective date for sort/group/jump/neighbor nav.
"""
existing = (await db.execute(
select(ImageProvenance.id).where(
ImageProvenance.image_record_id == image_id,
@@ -130,6 +137,18 @@ async def _ensure_provenance(
ImageProvenance.source_id == source_id,
)
)).scalar_one_or_none()
# Whether-or-not the provenance row already exists, ensure the
# image's primary_post_id is set so the gallery date-coalesce works.
# Idempotent: only writes when currently NULL.
if not dry_run:
await db.execute(
ImageRecord.__table__.update()
.where(ImageRecord.id == image_id)
.where(ImageRecord.primary_post_id.is_(None))
.values(primary_post_id=post_id)
)
if existing is not None:
return False
if dry_run:
+132
View File
@@ -133,3 +133,135 @@ async def test_get_image_with_tags_includes_integrity_status(db):
svc = GalleryService(db)
payload = await svc.get_image_with_tags(img.id)
assert payload["integrity_status"] == "ok"
async def _seed_image_with_post(
db, *, sha: str, image_created_at, post_date, artist_name="test-artist",
platform="patreon", external_post_id="42",
):
"""Helper: seed an Artist + Source + Post and one ImageRecord whose
primary_post_id points at that Post. Used for date-coalesce tests."""
from backend.app.models import Artist, Post, Source
artist = Artist(name=artist_name, slug=artist_name.lower().replace(" ", "-"))
db.add(artist)
await db.flush()
source = Source(
artist_id=artist.id, platform=platform,
url=f"https://www.{platform}.com/{artist.slug}",
)
db.add(source)
await db.flush()
post = Post(
source_id=source.id, external_post_id=external_post_id,
post_title="A Post", post_date=post_date,
)
db.add(post)
await db.flush()
img = ImageRecord(
path=f"/images/test/{sha[:8]}.jpg",
sha256=sha, size_bytes=1000, mime="image/jpeg",
width=100, height=100,
origin="imported_filesystem", integrity_status="unknown",
primary_post_id=post.id,
)
img.created_at = image_created_at
db.add(img)
await db.flush()
return img, post
@pytest.mark.asyncio
async def test_scroll_sorts_by_post_date_when_available(db):
"""Operator-flagged 2026-05-25: ~57k IR images all imported in the
same week sort by image.created_at and pile up in one month bucket.
Once primary_post_id is wired (via tag_apply phase 4), the gallery
should sort by Post.post_date instead, spreading them across the
actual publish years."""
base_import = _now()
# Image A: imported NOW, but post was made 2 years ago.
img_a, _ = await _seed_image_with_post(
db, sha="a" * 64,
image_created_at=base_import,
post_date=base_import - timedelta(days=730),
artist_name="Aria", external_post_id="A-1",
)
# Image B: imported NOW (1 min later), post made YESTERDAY.
img_b, _ = await _seed_image_with_post(
db, sha="b" * 64,
image_created_at=base_import - timedelta(minutes=1),
post_date=base_import - timedelta(days=1),
artist_name="Bea", external_post_id="B-1",
)
# Image C: filesystem-imported, no primary_post_id, created 5 days ago.
img_c = ImageRecord(
path="/images/test/c.jpg", sha256="c" * 64,
size_bytes=1000, mime="image/jpeg",
width=100, height=100,
origin="imported_filesystem", integrity_status="unknown",
)
img_c.created_at = base_import - timedelta(days=5)
db.add(img_c)
await db.flush()
svc = GalleryService(db)
page = await svc.scroll(cursor=None, limit=10)
# Effective-date order: B (yesterday) > C (5 days ago) > A (2 years ago)
assert [i.id for i in page.images] == [img_b.id, img_c.id, img_a.id]
# API exposes both fields explicitly so the UI can show "Posted X / Imported Y".
a_payload = next(i for i in page.images if i.id == img_a.id)
assert a_payload.posted_at is not None
assert a_payload.posted_at < a_payload.created_at
c_payload = next(i for i in page.images if i.id == img_c.id)
assert c_payload.posted_at is None
assert c_payload.effective_date == c_payload.created_at
@pytest.mark.asyncio
async def test_timeline_buckets_use_post_date_when_available(db):
"""Timeline group-by must follow the same effective_date rule so the
UI's year/month navigation surfaces publish-date buckets, not the
single FC-scan bucket all migrated images share."""
base = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
await _seed_image_with_post(
db, sha="1" * 64,
image_created_at=base,
post_date=datetime(2024, 3, 10, tzinfo=UTC),
artist_name="Carl", external_post_id="C-1",
)
await _seed_image_with_post(
db, sha="2" * 64,
image_created_at=base,
post_date=datetime(2024, 3, 11, tzinfo=UTC),
artist_name="Dee", external_post_id="D-1",
)
await _seed_image_with_post(
db, sha="3" * 64,
image_created_at=base,
post_date=datetime(2025, 9, 1, tzinfo=UTC),
artist_name="Eli", external_post_id="E-1",
)
svc = GalleryService(db)
buckets = await svc.timeline()
bucket_keys = {(b.year, b.month, b.count) for b in buckets}
# Two posts in 2024-03, one in 2025-09 — even though all imported in 2026-06.
assert (2024, 3, 2) in bucket_keys
assert (2025, 9, 1) in bucket_keys
# The FC-import bucket should NOT appear since all 3 images have post_date.
assert not any(b.year == 2026 and b.month == 6 for b in buckets)
@pytest.mark.asyncio
async def test_get_image_with_tags_includes_posted_at_when_present(db):
base = _now()
img, _ = await _seed_image_with_post(
db, sha="f" * 64,
image_created_at=base,
post_date=base - timedelta(days=365),
artist_name="Fred", external_post_id="F-1",
)
svc = GalleryService(db)
payload = await svc.get_image_with_tags(img.id)
assert payload["posted_at"] is not None
# Image's own created_at is still surfaced separately.
assert payload["created_at"] != payload["posted_at"]
+61
View File
@@ -227,6 +227,67 @@ async def test_image_posts_creates_source_post_provenance(db, tmp_path):
)).scalar_one()
assert prov_count == 1
# Phase 4 must also set ImageRecord.primary_post_id so the gallery's
# effective_date COALESCE can surface Post.post_date. Operator-flagged
# 2026-05-25: without this, IR-migrated images keep sorting by FC's
# scan date instead of the original publish date.
primary_post_id = (await db.execute(
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
)).scalar_one()
canonical_post_id = (await db.execute(
select(Post.id).where(Post.external_post_id == "10001")
)).scalar_one()
assert primary_post_id == canonical_post_id
@pytest.mark.asyncio
async def test_image_posts_primary_post_id_not_clobbered(db, tmp_path):
"""If the importer already set primary_post_id (e.g. a downloaded
image with a known provenance), phase 4 must NOT overwrite it when
re-running tag_apply against the IR migration. The existing
download-time linkage is the source of truth."""
sha = "9" * 64
await _seed_image(db, sha, suffix="9")
# Pre-set primary_post_id to a sentinel Post so we can detect a clobber.
img_id = (await db.execute(
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
)).scalar_one()
# Build an existing Source + Post for the sentinel.
art = Artist(name="Pre-existing", slug="pre-existing")
db.add(art)
await db.flush()
src = Source(
artist_id=art.id, platform="patreon",
url="https://www.patreon.com/pre-existing",
)
db.add(src)
await db.flush()
sentinel_post = Post(
source_id=src.id, external_post_id="sentinel-99",
post_title="Pre-existing",
)
db.add(sentinel_post)
await db.flush()
await db.execute(
ImageRecord.__table__.update()
.where(ImageRecord.id == img_id)
.values(primary_post_id=sentinel_post.id)
)
await db.commit()
_write_manifest(tmp_path, image_posts=[
{**_POST_ENTRY, "image_sha256s": [sha]},
])
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
# The migration created a NEW Post (external_post_id="10001") and a
# new ImageProvenance, but primary_post_id must still point at the
# original sentinel.
primary_post_id = (await db.execute(
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
)).scalar_one()
assert primary_post_id == sentinel_post.id
@pytest.mark.asyncio
async def test_image_posts_idempotent_on_rerun(db, tmp_path):