diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index b14d414..8de2ed8 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -42,6 +42,8 @@ async def scroll(): "width": i.width, "height": i.height, "created_at": i.created_at.isoformat(), + "posted_at": i.posted_at.isoformat() if i.posted_at else None, + "effective_date": i.effective_date.isoformat(), "thumbnail_url": i.thumbnail_url, "artist": i.artist, } diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 8337390..0a46fec 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -1,26 +1,34 @@ """Cursor-paginated gallery queries. -Cursor format: opaque base64-encoded ":". -Pagination key is (created_at DESC, id DESC) so we don't drift when new -imports arrive between page loads. Decoding rejects malformed cursors with -a ValueError; the API layer translates that to HTTP 400. +Cursor format: opaque base64-encoded ":". + +Pagination key is (effective_date DESC, id DESC) where effective_date is +COALESCE(post.post_date, image_record.created_at) so the gallery surfaces +images by ORIGINAL publish date when known, falling back to FC's scan +date. Important for migrated content: ~57k IR images scanned in a single +week would otherwise all share the same created_at and pile up in one +month bucket. The effective_date spreads them across the years they +were originally published. + +Decoding rejects malformed cursors with a ValueError; the API layer +translates that to HTTP 400. """ import base64 from dataclasses import dataclass from datetime import datetime -from sqlalchemy import and_, exists, func, or_, select +from sqlalchemy import Select, and_, exists, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from ..models import Artist, ImageProvenance, ImageRecord, Source, Tag +from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag from ..models.tag import image_tag CURSOR_SEPARATOR = "|" -def encode_cursor(created_at: datetime, image_id: int) -> str: - raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}" +def encode_cursor(effective_date: datetime, image_id: int) -> str: + raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}" return base64.urlsafe_b64encode(raw.encode()).decode() @@ -33,6 +41,26 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]: raise ValueError(f"invalid cursor: {cursor!r}") from exc +def _effective_date_col(): + """SQL expression: COALESCE(post.post_date, image_record.created_at). + + Used as the canonical sort/group/filter key across the gallery so + images backfilled with primary_post_id (e.g. via tag_apply phase 4) + surface at their original publish date, not their FC import date. + Images without a Post (or with Post.post_date NULL) fall back to + image_record.created_at and still order coherently against + post-attached ones. + """ + return func.coalesce(Post.post_date, ImageRecord.created_at) + + +def _outer_join_primary_post(stmt: Select) -> Select: + """LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE + above sees Post.post_date when available. Images without a post + survive the join as NULL on the Post side; COALESCE handles it.""" + return stmt.outerjoin(Post, Post.id == ImageRecord.primary_post_id) + + @dataclass(frozen=True) class GalleryImage: id: int @@ -41,7 +69,9 @@ class GalleryImage: mime: str width: int | None height: int | None - created_at: datetime + created_at: datetime # FC's row-insert time + effective_date: datetime # COALESCE(post.post_date, created_at) + posted_at: datetime | None # post.post_date if known, else None thumbnail_url: str artist: dict | None = None @@ -78,7 +108,7 @@ def _require_single_filter(tag_id, post_id, artist_id) -> None: def _provenance_clause(post_id, artist_id): """Correlated EXISTS clause (NOT a join) so an image with multiple matching provenance rows is returned exactly once and the - (created_at DESC, id DESC) cursor ordering is unaffected.""" + (effective_date DESC, id DESC) cursor ordering is unaffected.""" if post_id is not None: return exists().where( ImageProvenance.image_record_id == ImageRecord.id, @@ -125,7 +155,9 @@ class GalleryService: raise ValueError("limit must be between 1 and 200") _require_single_filter(tag_id, post_id, artist_id) - stmt = select(ImageRecord) + eff = _effective_date_col() + stmt = select(ImageRecord, Post.post_date, eff.label("eff")) + stmt = _outer_join_primary_post(stmt) if tag_id is not None: stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( image_tag.c.tag_id == tag_id @@ -138,34 +170,38 @@ class GalleryService: cur_ts, cur_id = decode_cursor(cursor) stmt = stmt.where( or_( - ImageRecord.created_at < cur_ts, - and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id), + eff < cur_ts, + and_(eff == cur_ts, ImageRecord.id < cur_id), ) ) - stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1) - rows = (await self.session.execute(stmt)).scalars().all() + stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() next_cursor = None if len(rows) > limit: - last = rows[limit - 1] - next_cursor = encode_cursor(last.created_at, last.id) + last_record, _last_posted_at, last_eff = rows[limit - 1] + next_cursor = encode_cursor(last_eff, last_record.id) rows = rows[:limit] - artists = await _artists_for(self.session, [r.id for r in rows]) + artists = await _artists_for( + self.session, [r[0].id for r in rows] + ) images = [ GalleryImage( - id=r.id, - path=r.path, - sha256=r.sha256, - mime=r.mime, - width=r.width, - height=r.height, - created_at=r.created_at, - thumbnail_url=thumbnail_url(r.sha256, r.mime), - artist=artists.get(r.id), + id=record.id, + path=record.path, + sha256=record.sha256, + mime=record.mime, + width=record.width, + height=record.height, + created_at=record.created_at, + effective_date=eff_date, + posted_at=posted_at, + thumbnail_url=thumbnail_url(record.sha256, record.mime), + artist=artists.get(record.id), ) - for r in rows + for record, posted_at, eff_date in rows ] return GalleryPage( images=images, @@ -179,11 +215,13 @@ class GalleryService: post_id: int | None = None, artist_id: int | None = None, ) -> list[TimelineBucket]: - year_col = func.date_part("year", ImageRecord.created_at).label("yr") - month_col = func.date_part("month", ImageRecord.created_at).label("mo") + eff = _effective_date_col() + year_col = func.date_part("year", eff).label("yr") + month_col = func.date_part("month", eff).label("mo") stmt = select( year_col, month_col, func.count(ImageRecord.id).label("cnt") ) + stmt = _outer_join_primary_post(stmt) _require_single_filter(tag_id, post_id, artist_id) if tag_id is not None: stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( @@ -201,14 +239,17 @@ class GalleryService: post_id: int | None = None, artist_id: int | None = None, ) -> str | None: """Returns a cursor that, when passed to scroll(), positions at the - first image of the given year-month. None if the bucket is empty. + first image of the given year-month (by effective_date, not + created_at). None if the bucket is empty. """ from sqlalchemy import extract - stmt = select(ImageRecord).where( - extract("year", ImageRecord.created_at) == year, - extract("month", ImageRecord.created_at) == month, + eff = _effective_date_col() + stmt = select(ImageRecord, eff.label("eff")).where( + extract("year", eff) == year, + extract("month", eff) == month, ) + stmt = _outer_join_primary_post(stmt) _require_single_filter(tag_id, post_id, artist_id) if tag_id is not None: stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( @@ -217,13 +258,14 @@ class GalleryService: prov = _provenance_clause(post_id, artist_id) if prov is not None: stmt = stmt.where(prov) - stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1) - first = (await self.session.execute(stmt)).scalar_one_or_none() + stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1) + first = (await self.session.execute(stmt)).first() if first is None: return None + record, eff_date = first # Cursor is exclusive; we encode a cursor with id+1 so the row itself # is the first result in the next scroll(). - return encode_cursor(first.created_at, first.id + 1) + return encode_cursor(eff_date, record.id + 1) async def get_image_with_tags(self, image_id: int) -> dict | None: record = await self.session.get(ImageRecord, image_id) @@ -236,6 +278,13 @@ class GalleryService: .order_by(Tag.kind.asc(), Tag.name.asc()) ) tags = (await self.session.execute(tag_stmt)).scalars().all() + # Fetch the canonical post.post_date for this image (if any) so + # the modal can show "Posted on " alongside import date. + posted_at = None + if record.primary_post_id is not None: + posted_at = (await self.session.execute( + select(Post.post_date).where(Post.id == record.primary_post_id) + )).scalar_one_or_none() neighbors = await self._neighbors(record) # Direct artist FK — used by the modal's ProvenancePanel as a # fallback when ImageProvenance is empty (i.e., filesystem- @@ -256,6 +305,7 @@ class GalleryService: "size_bytes": record.size_bytes, "integrity_status": record.integrity_status, "created_at": record.created_at.isoformat(), + "posted_at": posted_at.isoformat() if posted_at else None, "thumbnail_url": thumbnail_url(record.sha256, record.mime), "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", "artist": ( @@ -275,34 +325,41 @@ class GalleryService: } async def _neighbors(self, record: ImageRecord) -> dict: - prev_stmt = ( - select(ImageRecord.id) - .where( + # Compute the boundary image's effective_date in Python (one query + # below + the SELECT we already have on `record`) and use it for + # the neighbor comparison. Cheaper than re-deriving in SQL via + # correlated subquery. + boundary_eff = record.created_at + if record.primary_post_id is not None: + post_date = (await self.session.execute( + select(Post.post_date).where(Post.id == record.primary_post_id) + )).scalar_one_or_none() + if post_date is not None: + boundary_eff = post_date + + eff = _effective_date_col() + prev_stmt = _outer_join_primary_post( + select(ImageRecord.id).where( or_( - ImageRecord.created_at > record.created_at, + eff > boundary_eff, and_( - ImageRecord.created_at == record.created_at, + eff == boundary_eff, ImageRecord.id > record.id, ), ) ) - .order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc()) - .limit(1) - ) - next_stmt = ( - select(ImageRecord.id) - .where( + ).order_by(eff.asc(), ImageRecord.id.asc()).limit(1) + next_stmt = _outer_join_primary_post( + select(ImageRecord.id).where( or_( - ImageRecord.created_at < record.created_at, + eff < boundary_eff, and_( - ImageRecord.created_at == record.created_at, + eff == boundary_eff, ImageRecord.id < record.id, ), ) ) - .order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()) - .limit(1) - ) + ).order_by(eff.desc(), ImageRecord.id.desc()).limit(1) prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none() next_id = (await self.session.execute(next_stmt)).scalar_one_or_none() return {"prev_id": prev_id, "next_id": next_id} @@ -311,9 +368,11 @@ class GalleryService: def _group_by_year_month( images: list[GalleryImage], ) -> list[tuple[int, int, list[int]]]: + """Group by effective_date's year/month so migrated content surfaces + in the publish-date buckets, not the FC-scan-date bucket.""" groups: list[tuple[int, int, list[int]]] = [] for img in images: - y, m = img.created_at.year, img.created_at.month + y, m = img.effective_date.year, img.effective_date.month if groups and groups[-1][0] == y and groups[-1][1] == m: groups[-1][2].append(img.id) else: diff --git a/backend/app/services/migrators/tag_apply.py b/backend/app/services/migrators/tag_apply.py index cfcb585..3f15c88 100644 --- a/backend/app/services/migrators/tag_apply.py +++ b/backend/app/services/migrators/tag_apply.py @@ -122,7 +122,14 @@ async def _ensure_provenance( db: AsyncSession, *, image_id: int, post_id: int, source_id: int, dry_run: bool, ) -> bool: - """Returns True if a new ImageProvenance row was inserted.""" + """Returns True if a new ImageProvenance row was inserted. + + Also sets ImageRecord.primary_post_id to this post if the image + doesn't already have one — preserves any primary_post_id already + assigned at download time by the importer (don't clobber). This is + the linkage gallery_service.py uses to surface Post.post_date as + the image's effective date for sort/group/jump/neighbor nav. + """ existing = (await db.execute( select(ImageProvenance.id).where( ImageProvenance.image_record_id == image_id, @@ -130,6 +137,18 @@ async def _ensure_provenance( ImageProvenance.source_id == source_id, ) )).scalar_one_or_none() + + # Whether-or-not the provenance row already exists, ensure the + # image's primary_post_id is set so the gallery date-coalesce works. + # Idempotent: only writes when currently NULL. + if not dry_run: + await db.execute( + ImageRecord.__table__.update() + .where(ImageRecord.id == image_id) + .where(ImageRecord.primary_post_id.is_(None)) + .values(primary_post_id=post_id) + ) + if existing is not None: return False if dry_run: diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index ecba6bb..705747c 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -133,3 +133,135 @@ async def test_get_image_with_tags_includes_integrity_status(db): svc = GalleryService(db) payload = await svc.get_image_with_tags(img.id) assert payload["integrity_status"] == "ok" + + +async def _seed_image_with_post( + db, *, sha: str, image_created_at, post_date, artist_name="test-artist", + platform="patreon", external_post_id="42", +): + """Helper: seed an Artist + Source + Post and one ImageRecord whose + primary_post_id points at that Post. Used for date-coalesce tests.""" + from backend.app.models import Artist, Post, Source + artist = Artist(name=artist_name, slug=artist_name.lower().replace(" ", "-")) + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform=platform, + url=f"https://www.{platform}.com/{artist.slug}", + ) + db.add(source) + await db.flush() + post = Post( + source_id=source.id, external_post_id=external_post_id, + post_title="A Post", post_date=post_date, + ) + db.add(post) + await db.flush() + img = ImageRecord( + path=f"/images/test/{sha[:8]}.jpg", + sha256=sha, size_bytes=1000, mime="image/jpeg", + width=100, height=100, + origin="imported_filesystem", integrity_status="unknown", + primary_post_id=post.id, + ) + img.created_at = image_created_at + db.add(img) + await db.flush() + return img, post + + +@pytest.mark.asyncio +async def test_scroll_sorts_by_post_date_when_available(db): + """Operator-flagged 2026-05-25: ~57k IR images all imported in the + same week sort by image.created_at and pile up in one month bucket. + Once primary_post_id is wired (via tag_apply phase 4), the gallery + should sort by Post.post_date instead, spreading them across the + actual publish years.""" + base_import = _now() + # Image A: imported NOW, but post was made 2 years ago. + img_a, _ = await _seed_image_with_post( + db, sha="a" * 64, + image_created_at=base_import, + post_date=base_import - timedelta(days=730), + artist_name="Aria", external_post_id="A-1", + ) + # Image B: imported NOW (1 min later), post made YESTERDAY. + img_b, _ = await _seed_image_with_post( + db, sha="b" * 64, + image_created_at=base_import - timedelta(minutes=1), + post_date=base_import - timedelta(days=1), + artist_name="Bea", external_post_id="B-1", + ) + # Image C: filesystem-imported, no primary_post_id, created 5 days ago. + img_c = ImageRecord( + path="/images/test/c.jpg", sha256="c" * 64, + size_bytes=1000, mime="image/jpeg", + width=100, height=100, + origin="imported_filesystem", integrity_status="unknown", + ) + img_c.created_at = base_import - timedelta(days=5) + db.add(img_c) + await db.flush() + + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10) + # Effective-date order: B (yesterday) > C (5 days ago) > A (2 years ago) + assert [i.id for i in page.images] == [img_b.id, img_c.id, img_a.id] + + # API exposes both fields explicitly so the UI can show "Posted X / Imported Y". + a_payload = next(i for i in page.images if i.id == img_a.id) + assert a_payload.posted_at is not None + assert a_payload.posted_at < a_payload.created_at + c_payload = next(i for i in page.images if i.id == img_c.id) + assert c_payload.posted_at is None + assert c_payload.effective_date == c_payload.created_at + + +@pytest.mark.asyncio +async def test_timeline_buckets_use_post_date_when_available(db): + """Timeline group-by must follow the same effective_date rule so the + UI's year/month navigation surfaces publish-date buckets, not the + single FC-scan bucket all migrated images share.""" + base = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) + await _seed_image_with_post( + db, sha="1" * 64, + image_created_at=base, + post_date=datetime(2024, 3, 10, tzinfo=UTC), + artist_name="Carl", external_post_id="C-1", + ) + await _seed_image_with_post( + db, sha="2" * 64, + image_created_at=base, + post_date=datetime(2024, 3, 11, tzinfo=UTC), + artist_name="Dee", external_post_id="D-1", + ) + await _seed_image_with_post( + db, sha="3" * 64, + image_created_at=base, + post_date=datetime(2025, 9, 1, tzinfo=UTC), + artist_name="Eli", external_post_id="E-1", + ) + svc = GalleryService(db) + buckets = await svc.timeline() + bucket_keys = {(b.year, b.month, b.count) for b in buckets} + # Two posts in 2024-03, one in 2025-09 — even though all imported in 2026-06. + assert (2024, 3, 2) in bucket_keys + assert (2025, 9, 1) in bucket_keys + # The FC-import bucket should NOT appear since all 3 images have post_date. + assert not any(b.year == 2026 and b.month == 6 for b in buckets) + + +@pytest.mark.asyncio +async def test_get_image_with_tags_includes_posted_at_when_present(db): + base = _now() + img, _ = await _seed_image_with_post( + db, sha="f" * 64, + image_created_at=base, + post_date=base - timedelta(days=365), + artist_name="Fred", external_post_id="F-1", + ) + svc = GalleryService(db) + payload = await svc.get_image_with_tags(img.id) + assert payload["posted_at"] is not None + # Image's own created_at is still surfaced separately. + assert payload["created_at"] != payload["posted_at"] diff --git a/tests/test_tag_apply.py b/tests/test_tag_apply.py index cf1a5c2..d1385c1 100644 --- a/tests/test_tag_apply.py +++ b/tests/test_tag_apply.py @@ -227,6 +227,67 @@ async def test_image_posts_creates_source_post_provenance(db, tmp_path): )).scalar_one() assert prov_count == 1 + # Phase 4 must also set ImageRecord.primary_post_id so the gallery's + # effective_date COALESCE can surface Post.post_date. Operator-flagged + # 2026-05-25: without this, IR-migrated images keep sorting by FC's + # scan date instead of the original publish date. + primary_post_id = (await db.execute( + select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id) + )).scalar_one() + canonical_post_id = (await db.execute( + select(Post.id).where(Post.external_post_id == "10001") + )).scalar_one() + assert primary_post_id == canonical_post_id + + +@pytest.mark.asyncio +async def test_image_posts_primary_post_id_not_clobbered(db, tmp_path): + """If the importer already set primary_post_id (e.g. a downloaded + image with a known provenance), phase 4 must NOT overwrite it when + re-running tag_apply against the IR migration. The existing + download-time linkage is the source of truth.""" + sha = "9" * 64 + await _seed_image(db, sha, suffix="9") + # Pre-set primary_post_id to a sentinel Post so we can detect a clobber. + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + # Build an existing Source + Post for the sentinel. + art = Artist(name="Pre-existing", slug="pre-existing") + db.add(art) + await db.flush() + src = Source( + artist_id=art.id, platform="patreon", + url="https://www.patreon.com/pre-existing", + ) + db.add(src) + await db.flush() + sentinel_post = Post( + source_id=src.id, external_post_id="sentinel-99", + post_title="Pre-existing", + ) + db.add(sentinel_post) + await db.flush() + await db.execute( + ImageRecord.__table__.update() + .where(ImageRecord.id == img_id) + .values(primary_post_id=sentinel_post.id) + ) + await db.commit() + + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "image_sha256s": [sha]}, + ]) + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + # The migration created a NEW Post (external_post_id="10001") and a + # new ImageProvenance, but primary_post_id must still point at the + # original sentinel. + primary_post_id = (await db.execute( + select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id) + )).scalar_one() + assert primary_post_id == sentinel_post.id + @pytest.mark.asyncio async def test_image_posts_idempotent_on_rerun(db, tmp_path):