From c22f37d64dc95a94febf564aaf0379077a2c3c3c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 1 Jul 2026 10:46:09 -0400 Subject: [PATCH] feat(gallery): sort by earliest post date across all posts (new default) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gallery's newest/oldest sort keys off image_record.effective_date = COALESCE(primary post's post_date, created_at). The primary post is often the repost/download the file came from, so the grid led with download dates rather than when content was first posted (operator-flagged). Add a second materialized sort key, earliest_post_date = MIN(post_date) across ALL of an image's provenance posts (every post it appears in), else created_at — the original publish date. Mirrors the effective_date pattern so the sort stays a forward index scan. - alembic 0071: add earliest_post_date + index (DESC, id DESC); backfill created_at baseline then MIN over image_provenance ⋈ post. - importer: recompute earliest_post_date whenever a dated post is linked (MIN over the image's provenance, which now includes the just-added row). - gallery_service: new sorts posted_new / posted_old key off earliest_post_date; cursor + year/month grouping follow the active column transparently. - api: accept posted_new|posted_old; DEFAULT is now posted_new so the grid leads with original publish date. newest/oldest (effective_date) still available. - frontend: sort dropdown gains "Newest/Oldest post date" (default Newest post date); existing effective-date sorts relabelled "Newest/Oldest added". - tests: service test asserts posted_new/posted_old key off earliest_post_date; frontend default-sort omission test updated. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- .../0071_image_record_earliest_post_date.py | 80 +++++++++++++++++++ backend/app/api/gallery.py | 9 ++- backend/app/models/image_record.py | 10 +++ backend/app/services/gallery_service.py | 26 +++++- backend/app/services/importer.py | 20 ++++- .../components/gallery/GalleryFilterBar.vue | 10 ++- frontend/src/stores/gallery.js | 8 +- frontend/test/gallery.spec.js | 6 +- tests/test_gallery_service.py | 22 +++++ 9 files changed, 176 insertions(+), 15 deletions(-) create mode 100644 alembic/versions/0071_image_record_earliest_post_date.py diff --git a/alembic/versions/0071_image_record_earliest_post_date.py b/alembic/versions/0071_image_record_earliest_post_date.py new file mode 100644 index 0000000..b2e8f0c --- /dev/null +++ b/alembic/versions/0071_image_record_earliest_post_date.py @@ -0,0 +1,80 @@ +"""image_record.earliest_post_date: original-publish gallery sort key + index + +Revision ID: 0071 +Revises: 0070 +Create Date: 2026-07-01 + +effective_date (0035) keys off the PRIMARY post — which is often the repost / +download the file actually came from — and falls back to created_at, so the +gallery's default order surfaces download dates rather than when content was +first posted (operator-flagged 2026-07-01). Materialize a second sort key, +earliest_post_date = MIN(post_date) across ALL of an image's provenance posts +(every post it appears in), falling back to created_at only when no linked post +carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an +index range scan just like effective_date. + +Backfill mirrors 0035: created_at baseline, then override with the MIN over +image_provenance ⋈ post. New rows get the created_at-equivalent server default; +services/importer.py recomputes it whenever a dated post is linked. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0071" +down_revision: Union[str, None] = "0070" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable first so the backfill can populate before NOT NULL. + op.add_column( + "image_record", + sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True), + ) + # Baseline: download date. Set-based (no per-row binds) → immune to the + # 65535 bind-parameter ceiling regardless of library size. + op.execute( + """ + UPDATE image_record + SET earliest_post_date = created_at + """ + ) + # Override with the earliest post_date across EVERY post the image appears + # in (image_provenance is the many-to-many edge; ignore posts with no date). + op.execute( + """ + UPDATE image_record AS ir + SET earliest_post_date = sub.min_date + FROM ( + SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date + FROM image_provenance AS ip + JOIN post AS p ON p.id = ip.post_id + WHERE p.post_date IS NOT NULL + GROUP BY ip.image_record_id + ) AS sub + WHERE ir.id = sub.iid + """ + ) + op.alter_column( + "image_record", + "earliest_post_date", + nullable=False, + server_default=sa.text("now()"), + ) + # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC + # so the "post date" scroll is a forward index scan; raw SQL because + # alembic's column list doesn't express per-column DESC cleanly. + op.execute( + "CREATE INDEX ix_image_record_earliest_post_date " + "ON image_record (earliest_post_date DESC, id DESC)" + ) + + +def downgrade() -> None: + op.drop_index( + "ix_image_record_earliest_post_date", table_name="image_record" + ) + op.drop_column("image_record", "earliest_post_date") diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 678ab4a..1d290cf 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -44,7 +44,8 @@ def _parse_filters(): the image must match at least one tag from EACH group (groups ANDed). - `tag_not` is a comma-separated exclude list (image must carry none). - `media` is image|video; `sort` is newest|oldest; `platform` selects one + `media` is image|video; `sort` is newest|oldest|posted_new|posted_old + (default posted_new); `platform` selects one platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds (date_to is widened by a day so the whole day is covered by the service's @@ -67,8 +68,12 @@ def _parse_filters(): artist_id = int(artist_id_raw) if artist_id_raw else None media = request.args.get("media") media_type = media if media in ("image", "video") else None + # newest/oldest key off effective_date (primary post / download); posted_new/ + # posted_old off earliest_post_date (original publish across all posts). The + # default is posted_new so the grid leads with original publish date, not the + # download/repost the primary post points at (operator-flagged 2026-07-01). sort = request.args.get("sort") - sort = sort if sort in ("newest", "oldest") else "newest" + sort = sort if sort in ("newest", "oldest", "posted_new", "posted_old") else "posted_new" platform = request.args.get("platform") or None untagged = request.args.get("untagged") in ("1", "true", "yes") no_artist = request.args.get("no_artist") in ("1", "true", "yes") diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index fc459dd..32d6eaa 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -97,6 +97,16 @@ class ImageRecord(Base): effective_date: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + # Denormalized ORIGINAL-publish sort key (alembic 0071) = MIN(post_date) + # across ALL of the image's provenance posts, else created_at. effective_date + # above keys off the PRIMARY post (often the repost/download the file came + # from); this keys off the earliest publish across EVERY post the image + # appears in, so the gallery can sort by when content was first posted rather + # than when it was downloaded (operator-flagged 2026-07-01). Maintained by + # services/importer.py, recomputed whenever a dated post is linked. + earliest_post_date: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index d4e923b..e385459 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -55,6 +55,25 @@ def _effective_date_col(): return ImageRecord.effective_date +# Sort key -> the materialized column the gallery orders + cursors on. Both are +# indexed (DESC, id DESC), so every sort is a forward index range scan. +# newest/oldest → effective_date (primary post's date, else download) +# posted_new/_old → earliest_post_date (earliest publish across ALL posts) +_SORT_COLUMNS = { + "newest": ImageRecord.effective_date, + "oldest": ImageRecord.effective_date, + "posted_new": ImageRecord.earliest_post_date, + "posted_old": ImageRecord.earliest_post_date, +} +_ASCENDING_SORTS = {"oldest", "posted_old"} + + +def _sort_column(sort: str): + """The materialized date column a gallery sort orders/cursors on (falls back + to effective_date for any unknown sort).""" + return _SORT_COLUMNS.get(sort, ImageRecord.effective_date) + + def _outer_join_primary_post(stmt: Select) -> Select: """LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE above sees Post.post_date when available. Images without a post @@ -406,7 +425,10 @@ class GalleryService: tag_ids, post_id, artist_id, tag_or_groups, tag_exclude, ) - eff = _effective_date_col() + # eff is the ACTIVE sort column (effective_date or earliest_post_date); + # the cursor, ordering and year/month grouping all key off it, so the + # 'post date' sort paginates + buckets by original publish transparently. + eff = _sort_column(sort) stmt = select(ImageRecord, Post.post_date, eff.label("eff")) stmt = _outer_join_primary_post(stmt) stmt = _apply_scope( @@ -417,7 +439,7 @@ class GalleryService: date_from=date_from, date_to=date_to, ) - descending = sort != "oldest" + descending = sort not in _ASCENDING_SORTS if cursor: cur_ts, cur_id = decode_cursor(cursor) # The cursor is just (last eff, last id); the request's sort diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 70bc7c8..8afd219 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -17,7 +17,7 @@ from enum import StrEnum from pathlib import Path from PIL import Image -from sqlalchemy import select, update +from sqlalchemy import func, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -1411,6 +1411,24 @@ class Importer: # default (matches the old COALESCE(post_date, created_at) fallback). if record.primary_post_id == post.id and post.post_date is not None: record.effective_date = post.post_date + # earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this + # image's provenance posts, not just the primary — so the gallery can + # sort by original publish rather than the download/repost the primary + # points at. Recompute from provenance whenever a dated post is linked; + # the provenance row for THIS post was committed above, so the MIN + # includes it. Leaves the created_at default when no linked post is dated. + if post.post_date is not None: + earliest = self.session.execute( + select(func.min(Post.post_date)) + .select_from(ImageProvenance) + .join(Post, Post.id == ImageProvenance.post_id) + .where( + ImageProvenance.image_record_id == record.id, + Post.post_date.is_not(None), + ) + ).scalar_one_or_none() + if earliest is not None: + record.earliest_post_date = earliest self.session.flush() def _copy_to_library( diff --git a/frontend/src/components/gallery/GalleryFilterBar.vue b/frontend/src/components/gallery/GalleryFilterBar.vue index e13949a..f0a9b7e 100644 --- a/frontend/src/components/gallery/GalleryFilterBar.vue +++ b/frontend/src/components/gallery/GalleryFilterBar.vue @@ -142,9 +142,13 @@ const tagStore = useTagStore() const api = useApi() const router = useRouter() +// posted_* sort by earliest publish across ALL of an image's posts (original +// post date); newest/oldest sort by the primary post's date, else download date. const SORTS = [ - { title: 'Newest first', value: 'newest' }, - { title: 'Oldest first', value: 'oldest' }, + { title: 'Newest post date', value: 'posted_new' }, + { title: 'Oldest post date', value: 'posted_old' }, + { title: 'Newest added', value: 'newest' }, + { title: 'Oldest added', value: 'oldest' }, ] const selected = ref(null) @@ -175,7 +179,7 @@ const hasActiveFilters = computed(() => store.filter.tag_or.length > 0 || store.filter.artist_id != null || store.filter.media_type != null || - store.filter.sort !== 'newest' || + store.filter.sort !== 'posted_new' || store.filter.similar_to != null || hasRefineFilters.value ) diff --git a/frontend/src/stores/gallery.js b/frontend/src/stores/gallery.js index a0599bf..617cc73 100644 --- a/frontend/src/stores/gallery.js +++ b/frontend/src/stores/gallery.js @@ -23,7 +23,7 @@ export const useGalleryStore = defineStore('gallery', () => { const error = ref(null) const filter = ref({ tag_ids: [], artist_id: null, media_type: null, - sort: 'newest', post_id: null, + sort: 'posted_new', post_id: null, // #6 structured tag filter (AND-of-OR + exclude). tag_ids are the AND // "include" singletons (light editor + back-compat); tag_or is a list of // OR-groups (each group ORs, groups AND); tag_exclude is the NOT set. @@ -154,7 +154,7 @@ export const useGalleryStore = defineStore('gallery', () => { if (filter.value.tag_exclude.length) p.tag_not = filter.value.tag_exclude.join(',') if (filter.value.artist_id) p.artist_id = filter.value.artist_id if (filter.value.media_type) p.media = filter.value.media_type - if (filter.value.sort && filter.value.sort !== 'newest') p.sort = filter.value.sort + if (filter.value.sort && filter.value.sort !== 'posted_new') p.sort = filter.value.sort if (filter.value.platform) p.platform = filter.value.platform if (filter.value.untagged) p.untagged = '1' if (filter.value.no_artist) p.no_artist = '1' @@ -191,7 +191,7 @@ export const useGalleryStore = defineStore('gallery', () => { tag_exclude: _parseIds(q.tag_not), artist_id: _toId(q.artist_id), media_type: ['image', 'video'].includes(q.media) ? q.media : null, - sort: q.sort === 'oldest' ? 'oldest' : 'newest', + sort: ['newest', 'oldest', 'posted_new', 'posted_old'].includes(q.sort) ? q.sort : 'posted_new', post_id: _toId(q.post_id), platform: q.platform || null, untagged: _truthy(q.untagged), @@ -297,7 +297,7 @@ export function filterToQuery(f) { if (f.tag_exclude?.length) q.tag_not = f.tag_exclude.join(',') if (f.artist_id) q.artist_id = String(f.artist_id) if (f.media_type) q.media = f.media_type - if (f.sort && f.sort !== 'newest') q.sort = f.sort + if (f.sort && f.sort !== 'posted_new') q.sort = f.sort if (f.platform) q.platform = f.platform if (f.untagged) q.untagged = '1' if (f.no_artist) q.no_artist = '1' diff --git a/frontend/test/gallery.spec.js b/frontend/test/gallery.spec.js index ce533e3..1946e8e 100644 --- a/frontend/test/gallery.spec.js +++ b/frontend/test/gallery.spec.js @@ -43,7 +43,7 @@ describe('gallery store: composable filter', () => { expect(scroll).toContain('sort=oldest') }) - it('omits sort=newest and sends no filter params when empty', async () => { + it('omits the default sort and sends no filter params when empty', async () => { const s = useGalleryStore() const urls = [] stubFetch((url) => { urls.push(url); return { status: 200, body: EMPTY } }) @@ -197,7 +197,7 @@ describe('gallery store: composable filter', () => { describe('filterToQuery / cloneFilter', () => { it('serializes faceted params incl. platform sentinel and flags', () => { const q = filterToQuery({ - tag_ids: [3], artist_id: null, media_type: null, sort: 'newest', + tag_ids: [3], artist_id: null, media_type: null, sort: 'posted_new', platform: '__unsourced__', untagged: true, no_artist: false, date_from: '2024-01-01', date_to: null, }) @@ -207,7 +207,7 @@ describe('filterToQuery / cloneFilter', () => { expect(q.no_artist).toBeUndefined() expect(q.date_from).toBe('2024-01-01') expect(q.date_to).toBeUndefined() - expect(q.sort).toBeUndefined() // newest is the default → omitted + expect(q.sort).toBeUndefined() // posted_new is the default → omitted }) it('serializes similar_to', () => { diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index d4f754c..9c4a54b 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -73,6 +73,28 @@ async def test_scroll_returns_newest_first(db): assert page.images[0].created_at > page.images[-1].created_at +@pytest.mark.asyncio +async def test_scroll_sorts_by_earliest_post_date(db): + """posted_new/posted_old key off earliest_post_date (original publish across + all of an image's posts), NOT effective_date/created_at — so an old file that + was first posted recently sorts by that recent post date.""" + imgs = await _seed_images(db, 3) + now = _now() + # earliest_post_date order deliberately DIFFERS from created_at order (which + # _seed_images makes imgs[0] newest → imgs[2] oldest), proving the sort keys + # off earliest_post_date. + imgs[0].earliest_post_date = now - timedelta(days=1) # middle + imgs[1].earliest_post_date = now - timedelta(days=365) # oldest post + imgs[2].earliest_post_date = now - timedelta(hours=1) # newest post + await db.flush() + + svc = GalleryService(db) + newest = await svc.scroll(cursor=None, limit=10, sort="posted_new") + assert [i.id for i in newest.images] == [imgs[2].id, imgs[0].id, imgs[1].id] + oldest = await svc.scroll(cursor=None, limit=10, sort="posted_old") + assert [i.id for i in oldest.images] == [imgs[1].id, imgs[0].id, imgs[2].id] + + @pytest.mark.asyncio async def test_scroll_pagination(db): await _seed_images(db, 5)