bvandeusen
c9089b1d03
gallery_service._provenance_clause artist branch was correlating its bare Post reference to the outer query's primary_post_id outer-join, so the artist filter silently matched zero rows for images with no primary post. Alias Post inside the EXISTS subquery so SQLAlchemy adds it to the inner FROM rather than treating it as a correlated outer table. Five sidecar/import tests still asserted that a synthetic Source row appears after a filesystem import. Alembic 0030 retired that behavior; the Post sits null-source and the artist linkage lives on Post.artist_id. Updated test_sidecar_creates_provenance, test_reimport_same_post_idempotent, test_sidecar_artist_used_when_no_folder_artist, test_supersede_applies_new_file_sidecar, and test_apply_sidecar_recovers_from_integrity_error to assert post.source_id IS NULL + post.artist_id linkage instead.
413 lines
16 KiB
Python
413 lines
16 KiB
Python
"""Cursor-paginated gallery queries.
|
|
|
|
Cursor format: opaque base64-encoded "<iso8601_effective_date>:<image_id>".
|
|
|
|
Pagination key is (effective_date DESC, id DESC) where effective_date is
|
|
COALESCE(post.post_date, image_record.created_at) so the gallery surfaces
|
|
images by ORIGINAL publish date when known, falling back to FC's scan
|
|
date. Important for migrated content: ~57k IR images scanned in a single
|
|
week would otherwise all share the same created_at and pile up in one
|
|
month bucket. The effective_date spreads them across the years they
|
|
were originally published.
|
|
|
|
Decoding rejects malformed cursors with a ValueError; the API layer
|
|
translates that to HTTP 400.
|
|
"""
|
|
|
|
import base64
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import Select, and_, exists, func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy.orm import aliased
|
|
|
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
|
from ..models.tag import image_tag
|
|
|
|
CURSOR_SEPARATOR = "|"
|
|
|
|
|
|
def encode_cursor(effective_date: datetime, image_id: int) -> str:
|
|
raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}"
|
|
return base64.urlsafe_b64encode(raw.encode()).decode()
|
|
|
|
|
|
def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
|
try:
|
|
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
|
ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
|
|
return datetime.fromisoformat(ts_part), int(id_part)
|
|
except Exception as exc:
|
|
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
|
|
|
|
|
def _effective_date_col():
|
|
"""SQL expression: COALESCE(post.post_date, image_record.created_at).
|
|
|
|
Used as the canonical sort/group/filter key across the gallery so
|
|
images backfilled with primary_post_id (e.g. via tag_apply phase 4)
|
|
surface at their original publish date, not their FC import date.
|
|
Images without a Post (or with Post.post_date NULL) fall back to
|
|
image_record.created_at and still order coherently against
|
|
post-attached ones.
|
|
"""
|
|
return func.coalesce(Post.post_date, ImageRecord.created_at)
|
|
|
|
|
|
def _outer_join_primary_post(stmt: Select) -> Select:
|
|
"""LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
|
|
above sees Post.post_date when available. Images without a post
|
|
survive the join as NULL on the Post side; COALESCE handles it."""
|
|
return stmt.outerjoin(Post, Post.id == ImageRecord.primary_post_id)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GalleryImage:
|
|
id: int
|
|
path: str
|
|
sha256: str
|
|
mime: str
|
|
width: int | None
|
|
height: int | None
|
|
created_at: datetime # FC's row-insert time
|
|
effective_date: datetime # COALESCE(post.post_date, created_at)
|
|
posted_at: datetime | None # post.post_date if known, else None
|
|
thumbnail_url: str
|
|
artist: dict | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GalleryPage:
|
|
images: list[GalleryImage]
|
|
next_cursor: str | None
|
|
date_groups: list[tuple[int, int, list[int]]] # (year, month, [image_id...])
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TimelineBucket:
|
|
year: int
|
|
month: int
|
|
count: int
|
|
|
|
|
|
def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str:
|
|
"""Return the URL to fetch a thumbnail.
|
|
|
|
Prefers the stored thumbnail_path verbatim — Quart serves /images/*
|
|
1:1 from the volume (frontend.py:20-36), so the URL IS the disk
|
|
path. Falls back to deriving from (sha256, mime) only when the
|
|
record's thumbnail_path is NULL (thumbnailer hasn't run yet); that
|
|
URL will 404 until backfill catches it, same as before the path
|
|
was tracked.
|
|
|
|
Pre-2026-05-30 this was derived only from (sha256, mime), which
|
|
disagreed with the actual on-disk extension when the thumbnailer
|
|
chose its format from transparency rather than MIME — every PNG
|
|
source without alpha (extension was .jpg on disk) and every WebP
|
|
source with alpha (extension was .png on disk) silently 404'd
|
|
despite the thumbnail file existing.
|
|
"""
|
|
if thumbnail_path:
|
|
return thumbnail_path
|
|
# Fallback for records with no thumbnail recorded yet — preserves
|
|
# prior behavior (URL exists but 404s until backfill regenerates).
|
|
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
|
|
bucket = sha256_hex[:3]
|
|
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
|
|
|
|
|
|
def _require_single_filter(tag_id, post_id, artist_id) -> None:
|
|
if sum(x is not None for x in (tag_id, post_id, artist_id)) > 1:
|
|
raise ValueError(
|
|
"tag_id, post_id, artist_id are mutually exclusive"
|
|
)
|
|
|
|
|
|
def _provenance_clause(post_id, artist_id):
|
|
"""Correlated EXISTS clause (NOT a join) so an image with multiple
|
|
matching provenance rows is returned exactly once and the
|
|
(effective_date DESC, id DESC) cursor ordering is unaffected."""
|
|
if post_id is not None:
|
|
return exists().where(
|
|
ImageProvenance.image_record_id == ImageRecord.id,
|
|
ImageProvenance.post_id == post_id,
|
|
)
|
|
if artist_id is not None:
|
|
# Use Post.artist_id (alembic 0030 denormalized column) instead
|
|
# of joining through ImageProvenance.source_id → Source.artist_id.
|
|
# The denormalization is the always-present linkage; the source
|
|
# path now drops NULL-source provenance rows (filesystem-imported
|
|
# content) which would otherwise vanish from artist-filtered
|
|
# gallery views.
|
|
# ALIAS Post: the gallery query outer-joins Post on
|
|
# ImageRecord.primary_post_id (`_outer_join_primary_post`).
|
|
# SQLAlchemy would otherwise correlate a bare `Post` reference
|
|
# in this EXISTS subquery to that outer Post (which is NULL for
|
|
# images with no primary post), and the filter would silently
|
|
# match nothing.
|
|
post_inner = aliased(Post)
|
|
return exists().where(
|
|
ImageProvenance.image_record_id == ImageRecord.id,
|
|
ImageProvenance.post_id == post_inner.id,
|
|
post_inner.artist_id == artist_id,
|
|
)
|
|
return None
|
|
|
|
|
|
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
|
"""Map image_id -> {"name","slug"} via the canonical
|
|
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
|
if not image_ids:
|
|
return {}
|
|
stmt = (
|
|
select(ImageRecord.id, Artist.name, Artist.slug)
|
|
.join(Artist, Artist.id == ImageRecord.artist_id)
|
|
.where(ImageRecord.id.in_(image_ids))
|
|
)
|
|
return {
|
|
img_id: {"name": name, "slug": slug}
|
|
for img_id, name, slug in (await session.execute(stmt)).all()
|
|
}
|
|
|
|
|
|
class GalleryService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
async def scroll(
|
|
self,
|
|
cursor: str | None,
|
|
limit: int = 50,
|
|
tag_id: int | None = None,
|
|
post_id: int | None = None,
|
|
artist_id: int | None = None,
|
|
) -> GalleryPage:
|
|
if limit < 1 or limit > 200:
|
|
raise ValueError("limit must be between 1 and 200")
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
|
|
eff = _effective_date_col()
|
|
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
|
stmt = _outer_join_primary_post(stmt)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
|
|
if cursor:
|
|
cur_ts, cur_id = decode_cursor(cursor)
|
|
stmt = stmt.where(
|
|
or_(
|
|
eff < cur_ts,
|
|
and_(eff == cur_ts, ImageRecord.id < cur_id),
|
|
)
|
|
)
|
|
|
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1)
|
|
rows = (await self.session.execute(stmt)).all()
|
|
|
|
next_cursor = None
|
|
if len(rows) > limit:
|
|
last_record, _last_posted_at, last_eff = rows[limit - 1]
|
|
next_cursor = encode_cursor(last_eff, last_record.id)
|
|
rows = rows[:limit]
|
|
|
|
artists = await _artists_for(
|
|
self.session, [r[0].id for r in rows]
|
|
)
|
|
images = [
|
|
GalleryImage(
|
|
id=record.id,
|
|
path=record.path,
|
|
sha256=record.sha256,
|
|
mime=record.mime,
|
|
width=record.width,
|
|
height=record.height,
|
|
created_at=record.created_at,
|
|
effective_date=eff_date,
|
|
posted_at=posted_at,
|
|
thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
|
artist=artists.get(record.id),
|
|
)
|
|
for record, posted_at, eff_date in rows
|
|
]
|
|
return GalleryPage(
|
|
images=images,
|
|
next_cursor=next_cursor,
|
|
date_groups=_group_by_year_month(images),
|
|
)
|
|
|
|
async def timeline(
|
|
self,
|
|
tag_id: int | None = None,
|
|
post_id: int | None = None,
|
|
artist_id: int | None = None,
|
|
) -> list[TimelineBucket]:
|
|
eff = _effective_date_col()
|
|
year_col = func.date_part("year", eff).label("yr")
|
|
month_col = func.date_part("month", eff).label("mo")
|
|
stmt = select(
|
|
year_col, month_col, func.count(ImageRecord.id).label("cnt")
|
|
)
|
|
stmt = _outer_join_primary_post(stmt)
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
|
rows = (await self.session.execute(stmt)).all()
|
|
return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows]
|
|
|
|
async def jump_cursor(
|
|
self, year: int, month: int, tag_id: int | None = None,
|
|
post_id: int | None = None, artist_id: int | None = None,
|
|
) -> str | None:
|
|
"""Returns a cursor that, when passed to scroll(), positions at the
|
|
first image of the given year-month (by effective_date, not
|
|
created_at). None if the bucket is empty.
|
|
"""
|
|
from sqlalchemy import extract
|
|
|
|
eff = _effective_date_col()
|
|
stmt = select(ImageRecord, eff.label("eff")).where(
|
|
extract("year", eff) == year,
|
|
extract("month", eff) == month,
|
|
)
|
|
stmt = _outer_join_primary_post(stmt)
|
|
_require_single_filter(tag_id, post_id, artist_id)
|
|
if tag_id is not None:
|
|
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
|
image_tag.c.tag_id == tag_id
|
|
)
|
|
prov = _provenance_clause(post_id, artist_id)
|
|
if prov is not None:
|
|
stmt = stmt.where(prov)
|
|
stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
|
first = (await self.session.execute(stmt)).first()
|
|
if first is None:
|
|
return None
|
|
record, eff_date = first
|
|
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
|
|
# is the first result in the next scroll().
|
|
return encode_cursor(eff_date, record.id + 1)
|
|
|
|
async def get_image_with_tags(self, image_id: int) -> dict | None:
|
|
record = await self.session.get(ImageRecord, image_id)
|
|
if record is None:
|
|
return None
|
|
tag_stmt = (
|
|
select(Tag)
|
|
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
|
.where(image_tag.c.image_record_id == image_id)
|
|
.order_by(Tag.kind.asc(), Tag.name.asc())
|
|
)
|
|
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
|
# Fetch the canonical post.post_date for this image (if any) so
|
|
# the modal can show "Posted on <date>" alongside import date.
|
|
posted_at = None
|
|
if record.primary_post_id is not None:
|
|
posted_at = (await self.session.execute(
|
|
select(Post.post_date).where(Post.id == record.primary_post_id)
|
|
)).scalar_one_or_none()
|
|
neighbors = await self._neighbors(record)
|
|
# Direct artist FK — used by the modal's ProvenancePanel as a
|
|
# fallback when ImageProvenance is empty (i.e., filesystem-
|
|
# imported images without a post-track provenance row). The
|
|
# source of truth for richer post-level data is still
|
|
# ImageProvenance/Post; this is just the "we at least know who
|
|
# made it" line.
|
|
artist = None
|
|
if record.artist_id is not None:
|
|
artist = await self.session.get(Artist, record.artist_id)
|
|
return {
|
|
"id": record.id,
|
|
"path": record.path,
|
|
"sha256": record.sha256,
|
|
"mime": record.mime,
|
|
"width": record.width,
|
|
"height": record.height,
|
|
"size_bytes": record.size_bytes,
|
|
"integrity_status": record.integrity_status,
|
|
"created_at": record.created_at.isoformat(),
|
|
"posted_at": posted_at.isoformat() if posted_at else None,
|
|
"thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
|
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
|
"artist": (
|
|
{"id": artist.id, "name": artist.name, "slug": artist.slug}
|
|
if artist is not None else None
|
|
),
|
|
"tags": [
|
|
{
|
|
"id": t.id,
|
|
"name": t.name,
|
|
"kind": t.kind.value if hasattr(t.kind, "value") else t.kind,
|
|
"fandom_id": t.fandom_id,
|
|
}
|
|
for t in tags
|
|
],
|
|
"neighbors": neighbors,
|
|
}
|
|
|
|
async def _neighbors(self, record: ImageRecord) -> dict:
|
|
# Compute the boundary image's effective_date in Python (one query
|
|
# below + the SELECT we already have on `record`) and use it for
|
|
# the neighbor comparison. Cheaper than re-deriving in SQL via
|
|
# correlated subquery.
|
|
boundary_eff = record.created_at
|
|
if record.primary_post_id is not None:
|
|
post_date = (await self.session.execute(
|
|
select(Post.post_date).where(Post.id == record.primary_post_id)
|
|
)).scalar_one_or_none()
|
|
if post_date is not None:
|
|
boundary_eff = post_date
|
|
|
|
eff = _effective_date_col()
|
|
prev_stmt = _outer_join_primary_post(
|
|
select(ImageRecord.id).where(
|
|
or_(
|
|
eff > boundary_eff,
|
|
and_(
|
|
eff == boundary_eff,
|
|
ImageRecord.id > record.id,
|
|
),
|
|
)
|
|
)
|
|
).order_by(eff.asc(), ImageRecord.id.asc()).limit(1)
|
|
next_stmt = _outer_join_primary_post(
|
|
select(ImageRecord.id).where(
|
|
or_(
|
|
eff < boundary_eff,
|
|
and_(
|
|
eff == boundary_eff,
|
|
ImageRecord.id < record.id,
|
|
),
|
|
)
|
|
)
|
|
).order_by(eff.desc(), ImageRecord.id.desc()).limit(1)
|
|
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
|
|
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
|
|
return {"prev_id": prev_id, "next_id": next_id}
|
|
|
|
|
|
def _group_by_year_month(
|
|
images: list[GalleryImage],
|
|
) -> list[tuple[int, int, list[int]]]:
|
|
"""Group by effective_date's year/month so migrated content surfaces
|
|
in the publish-date buckets, not the FC-scan-date bucket."""
|
|
groups: list[tuple[int, int, list[int]]] = []
|
|
for img in images:
|
|
y, m = img.effective_date.year, img.effective_date.month
|
|
if groups and groups[-1][0] == y and groups[-1][1] == m:
|
|
groups[-1][2].append(img.id)
|
|
else:
|
|
groups.append((y, m, [img.id]))
|
|
return groups
|