bvandeusen
e05e0b9f37
The gallery cursored on COALESCE(post.post_date, image_record.created_at) across the Post outer join — an expression spanning two tables that no index can serve, so every /scroll sorted a large slice of the library (and the old frontend fired ten serially). Materialize it: - image_record.effective_date column + ix_image_record_effective_date (effective_date DESC, id DESC); alembic 0035 backfills COALESCE(primary post's post_date, created_at) for existing rows. - gallery_service._effective_date_col() now returns the column, so scroll / timeline / jump / neighbors all order off the index instead of re-deriving the COALESCE. _neighbors reads record.effective_date directly (drops an extra Post lookup). - importer._apply_sidecar maintains it: when a primary post with a date is linked, effective_date = post.post_date; plain inserts keep the created_at-equivalent server default. Tests: sidecar import asserts effective_date == post.post_date; gallery ordering/timeline/jump test seeds set effective_date alongside created_at. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
407 lines
16 KiB
Python
407 lines
16 KiB
Python
"""Cursor-paginated gallery queries.
|
|
|
|
Cursor format: opaque base64-encoded "<iso8601_effective_date>:<image_id>".
|
|
|
|
Pagination key is (effective_date DESC, id DESC) where effective_date is
|
|
COALESCE(post.post_date, image_record.created_at) so the gallery surfaces
|
|
images by ORIGINAL publish date when known, falling back to FC's scan
|
|
date. Important for migrated content: ~57k IR images scanned in a single
|
|
week would otherwise all share the same created_at and pile up in one
|
|
month bucket. The effective_date spreads them across the years they
|
|
were originally published.
|
|
|
|
Decoding rejects malformed cursors with a ValueError; the API layer
|
|
translates that to HTTP 400.
|
|
"""
|
|
|
|
import base64
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import Select, and_, exists, func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy.orm import aliased
|
|
|
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
|
from ..models.tag import image_tag
|
|
|
|
CURSOR_SEPARATOR = "|"
|
|
|
|
|
|
def encode_cursor(effective_date: datetime, image_id: int) -> str:
|
|
raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}"
|
|
return base64.urlsafe_b64encode(raw.encode()).decode()
|
|
|
|
|
|
def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
|
try:
|
|
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
|
ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
|
|
return datetime.fromisoformat(ts_part), int(id_part)
|
|
except Exception as exc:
|
|
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
|
|
|
|
|
def _effective_date_col():
|
|
"""The materialized gallery sort key: image_record.effective_date
|
|
(alembic 0035) = COALESCE(primary post's post_date, created_at),
|
|
maintained at write time by the importer.
|
|
|
|
Canonical sort/group/filter key across the gallery so images attached
|
|
to a post surface at their original publish date, not their FC import
|
|
date — and, now that it's a single indexed column rather than a
|
|
COALESCE across the Post outer join, the cursor scroll is an index
|
|
range scan instead of a full re-sort per page.
|
|
"""
|
|
return ImageRecord.effective_date
|
|
|
|
|
|
def _outer_join_primary_post(stmt: Select) -> Select:
|
|
"""LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
|
|
above sees Post.post_date when available. Images without a post
|
|
survive the join as NULL on the Post side; COALESCE handles it."""
|
|
return stmt.outerjoin(Post, Post.id == ImageRecord.primary_post_id)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GalleryImage:
|
|
id: int
|
|
path: str
|
|
sha256: str
|
|
mime: str
|
|
width: int | None
|
|
height: int | None
|
|
created_at: datetime # FC's row-insert time
|
|
effective_date: datetime # COALESCE(post.post_date, created_at)
|
|
posted_at: datetime | None # post.post_date if known, else None
|
|
thumbnail_url: str
|
|
artist: dict | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GalleryPage:
|
|
images: list[GalleryImage]
|
|
next_cursor: str | None
|
|
date_groups: list[tuple[int, int, list[int]]] # (year, month, [image_id...])
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TimelineBucket:
|
|
year: int
|
|
month: int
|
|
count: int
|
|
|
|
|
|
def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str:
|
|
"""Return the URL to fetch a thumbnail.
|
|
|
|
Prefers the stored thumbnail_path verbatim — Quart serves /images/*
|
|
1:1 from the volume (frontend.py:20-36), so the URL IS the disk
|
|
path. Falls back to deriving from (sha256, mime) only when the
|
|
record's thumbnail_path is NULL (thumbnailer hasn't run yet); that
|
|
URL will 404 until backfill catches it, same as before the path
|
|
was tracked.
|
|
|
|
Pre-2026-05-30 this was derived only from (sha256, mime), which
|
|
disagreed with the actual on-disk extension when the thumbnailer
|
|
chose its format from transparency rather than MIME — every PNG
|
|
source without alpha (extension was .jpg on disk) and every WebP
|
|
source with alpha (extension was .png on disk) silently 404'd
|
|
despite the thumbnail file existing.
|
|
"""
|
|
if thumbnail_path:
|
|
return thumbnail_path
|
|
# Fallback for records with no thumbnail recorded yet — preserves
|
|
# prior behavior (URL exists but 404s until backfill regenerates).
|
|
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
|
|
bucket = sha256_hex[:3]
|
|
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
|
|
|
|
|
|
def _require_single_filter(tag_id, post_id, artist_id) -> None:
|
|
if sum(x is not None for x in (tag_id, post_id, artist_id)) > 1:
|
|
raise ValueError(
|
|
"tag_id, post_id, artist_id are mutually exclusive"
|
|
)
|
|
|
|
|
|
def _provenance_clause(post_id, artist_id):
|
|
"""Correlated EXISTS clause (NOT a join) so an image with multiple
|
|
matching provenance rows is returned exactly once and the
|
|
(effective_date DESC, id DESC) cursor ordering is unaffected."""
|
|
if post_id is not None:
|
|
return exists().where(
|
|
ImageProvenance.image_record_id == ImageRecord.id,
|
|
ImageProvenance.post_id == post_id,
|
|
)
|
|
if artist_id is not None:
|
|
# Use Post.artist_id (alembic 0030 denormalized column) instead
|
|
# of joining through ImageProvenance.source_id → Source.artist_id.
|
|
# The denormalization is the always-present linkage; the source
|
|
# path now drops NULL-source provenance rows (filesystem-imported
|
|
# content) which would otherwise vanish from artist-filtered
|
|
# gallery views.
|
|
# ALIAS Post: the gallery query outer-joins Post on
|
|
# ImageRecord.primary_post_id (`_outer_join_primary_post`).
|
|
# SQLAlchemy would otherwise correlate a bare `Post` reference
|
|
# in this EXISTS subquery to that outer Post (which is NULL for
|
|
# images with no primary post), and the filter would silently
|
|
# match nothing.
|
|
post_inner = aliased(Post)
|
|
return exists().where(
|
|
ImageProvenance.image_record_id == ImageRecord.id,
|
|
ImageProvenance.post_id == post_inner.id,
|
|
post_inner.artist_id == artist_id,
|
|
)
|
|
return None
|
|
|
|
|
|
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
|
"""Map image_id -> {"name","slug"} via the canonical
|
|
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
|
if not image_ids:
|
|
return {}
|
|
stmt = (
|
|
select(ImageRecord.id, Artist.name, Artist.slug)
|
|
.join(Artist, Artist.id == ImageRecord.artist_id)
|
|
.where(ImageRecord.id.in_(image_ids))
|
|
)
|
|
return {
|
|
img_id: {"name": name, "slug": slug}
|
|
for img_id, name, slug in (await session.execute(stmt)).all()
|
|
}
|
|
|
|
|
|
class GalleryService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def scroll(
|
|
self,
|
|
cursor: str | None,
|
|
limit: int = 50,
|
|
tag_id: int | None = None,
|
|
post_id: int | None = None,
|
|
artist_id: int | None = None,
|
|
) -> GalleryPage:
|
|
if limit < 1 or limit > 200:
|
|
raise ValueError("limit must be between 1 and 200")
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
|
|
eff = _effective_date_col()
|
|
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
|
stmt = _outer_join_primary_post(stmt)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
|
|
if cursor:
|
|
cur_ts, cur_id = decode_cursor(cursor)
|
|
stmt = stmt.where(
|
|
or_(
|
|
eff < cur_ts,
|
|
and_(eff == cur_ts, ImageRecord.id < cur_id),
|
|
)
|
|
)
|
|
|
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
next_cursor = None
|
|
if len(rows) > limit:
|
|
last_record, _last_posted_at, last_eff = rows[limit - 1]
|
|
next_cursor = encode_cursor(last_eff, last_record.id)
|
|
rows = rows[:limit]
|
|
|
|
artists = await _artists_for(
|
|
self.session, [r[0].id for r in rows]
|
|
)
|
|
images = [
|
|
GalleryImage(
|
|
id=record.id,
|
|
path=record.path,
|
|
sha256=record.sha256,
|
|
mime=record.mime,
|
|
width=record.width,
|
|
height=record.height,
|
|
created_at=record.created_at,
|
|
effective_date=eff_date,
|
|
posted_at=posted_at,
|
|
thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
|
artist=artists.get(record.id),
|
|
)
|
|
for record, posted_at, eff_date in rows
|
|
]
|
|
return GalleryPage(
|
|
images=images,
|
|
next_cursor=next_cursor,
|
|
date_groups=_group_by_year_month(images),
|
|
)
|
|
|
|
async def timeline(
|
|
self,
|
|
tag_id: int | None = None,
|
|
post_id: int | None = None,
|
|
artist_id: int | None = None,
|
|
) -> list[TimelineBucket]:
|
|
eff = _effective_date_col()
|
|
year_col = func.date_part("year", eff).label("yr")
|
|
month_col = func.date_part("month", eff).label("mo")
|
|
stmt = select(
|
|
year_col, month_col, func.count(ImageRecord.id).label("cnt")
|
|
)
|
|
stmt = _outer_join_primary_post(stmt)
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
|
rows = (await self.session.execute(stmt)).all()
|
|
return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows]
|
|
|
|
async def jump_cursor(
|
|
self, year: int, month: int, tag_id: int | None = None,
|
|
post_id: int | None = None, artist_id: int | None = None,
|
|
) -> str | None:
|
|
"""Returns a cursor that, when passed to scroll(), positions at the
|
|
first image of the given year-month (by effective_date, not
|
|
created_at). None if the bucket is empty.
|
|
"""
|
|
from sqlalchemy import extract
|
|
|
|
eff = _effective_date_col()
|
|
stmt = select(ImageRecord, eff.label("eff")).where(
|
|
extract("year", eff) == year,
|
|
extract("month", eff) == month,
|
|
)
|
|
stmt = _outer_join_primary_post(stmt)
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
|
first = (await self.session.execute(stmt)).first()
|
|
if first is None:
|
|
return None
|
|
record, eff_date = first
|
|
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
|
|
# is the first result in the next scroll().
|
|
return encode_cursor(eff_date, record.id + 1)
|
|
|
|
async def get_image_with_tags(self, image_id: int) -> dict | None:
|
|
record = await self.session.get(ImageRecord, image_id)
|
|
if record is None:
|
|
return None
|
|
tag_stmt = (
|
|
select(Tag)
|
|
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
|
.where(image_tag.c.image_record_id == image_id)
|
|
.order_by(Tag.kind.asc(), Tag.name.asc())
|
|
)
|
|
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
|
# Fetch the canonical post.post_date for this image (if any) so
|
|
# the modal can show "Posted on <date>" alongside import date.
|
|
posted_at = None
|
|
if record.primary_post_id is not None:
|
|
posted_at = (await self.session.execute(
|
|
select(Post.post_date).where(Post.id == record.primary_post_id)
|
|
)).scalar_one_or_none()
|
|
neighbors = await self._neighbors(record)
|
|
# Direct artist FK — used by the modal's ProvenancePanel as a
|
|
# fallback when ImageProvenance is empty (i.e., filesystem-
|
|
# imported images without a post-track provenance row). The
|
|
# source of truth for richer post-level data is still
|
|
# ImageProvenance/Post; this is just the "we at least know who
|
|
# made it" line.
|
|
artist = None
|
|
if record.artist_id is not None:
|
|
artist = await self.session.get(Artist, record.artist_id)
|
|
return {
|
|
"id": record.id,
|
|
"path": record.path,
|
|
"sha256": record.sha256,
|
|
"mime": record.mime,
|
|
"width": record.width,
|
|
"height": record.height,
|
|
"size_bytes": record.size_bytes,
|
|
"integrity_status": record.integrity_status,
|
|
"created_at": record.created_at.isoformat(),
|
|
"posted_at": posted_at.isoformat() if posted_at else None,
|
|
"thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
|
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
|
"artist": (
|
|
{"id": artist.id, "name": artist.name, "slug": artist.slug}
|
|
if artist is not None else None
|
|
),
|
|
"tags": [
|
|
{
|
|
"id": t.id,
|
|
"name": t.name,
|
|
"kind": t.kind.value if hasattr(t.kind, "value") else t.kind,
|
|
"fandom_id": t.fandom_id,
|
|
}
|
|
for t in tags
|
|
],
|
|
"neighbors": neighbors,
|
|
}
|
|
|
|
async def _neighbors(self, record: ImageRecord) -> dict:
|
|
# The boundary image's sort key is materialized on the row now
|
|
# (alembic 0035) — read it directly instead of re-deriving COALESCE
|
|
# via an extra Post lookup.
|
|
boundary_eff = record.effective_date
|
|
|
|
eff = _effective_date_col()
|
|
prev_stmt = _outer_join_primary_post(
|
|
select(ImageRecord.id).where(
|
|
or_(
|
|
eff > boundary_eff,
|
|
and_(
|
|
eff == boundary_eff,
|
|
ImageRecord.id > record.id,
|
|
),
|
|
)
|
|
)
|
|
).order_by(eff.asc(), ImageRecord.id.asc()).limit(1)
|
|
next_stmt = _outer_join_primary_post(
|
|
select(ImageRecord.id).where(
|
|
or_(
|
|
eff < boundary_eff,
|
|
and_(
|
|
eff == boundary_eff,
|
|
ImageRecord.id < record.id,
|
|
),
|
|
)
|
|
)
|
|
).order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
|
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
|
|
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
|
|
return {"prev_id": prev_id, "next_id": next_id}
|
|
|
|
|
|
def _group_by_year_month(
|
|
images: list[GalleryImage],
|
|
) -> list[tuple[int, int, list[int]]]:
|
|
"""Group by effective_date's year/month so migrated content surfaces
|
|
in the publish-date buckets, not the FC-scan-date bucket."""
|
|
groups: list[tuple[int, int, list[int]]] = []
|
|
for img in images:
|
|
y, m = img.effective_date.year, img.effective_date.month
|
|
if groups and groups[-1][0] == y and groups[-1][1] == m:
|
|
groups[-1][2].append(img.id)
|
|
else:
|
|
groups.append((y, m, [img.id]))
|
|
return groups
|