diff --git a/.gitignore b/.gitignore index 06d2c93..10d4d4b 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,9 @@ Thumbs.db # Claude Code per-user local overrides (shared .claude/settings.json is OK to commit) .claude/settings.local.json +# Transient scheduler lock/state (committed by accident in 3f30327) +.claude/scheduled_tasks.lock +.claude/scheduled_tasks*.json # Alembic / DB scratch alembic/versions/__pycache__/ diff --git a/alembic/versions/0035_image_record_effective_date.py b/alembic/versions/0035_image_record_effective_date.py new file mode 100644 index 0000000..586cf51 --- /dev/null +++ b/alembic/versions/0035_image_record_effective_date.py @@ -0,0 +1,70 @@ +"""image_record.effective_date: materialized gallery sort key + index + +Revision ID: 0035 +Revises: 0034 +Create Date: 2026-06-04 + +The gallery ordered/cursored on COALESCE(post.post_date, +image_record.created_at) across the Post outer join. That expression spans +two tables, so no index can serve it — every /scroll sorted a large slice +of the library, and the frontend fired ten of them serially per initial +load. Materialize the value into image_record.effective_date and index +(effective_date DESC, id DESC) so the cursor scroll is an index range scan. + +Backfill = COALESCE(primary post's post_date, created_at) so existing rows +keep their exact ordering. New rows get the created_at-equivalent server +default; services/importer.py overrides it with the post's date when a +primary post with a date is linked. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0035" +down_revision: Union[str, None] = "0034" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable first so the backfill can populate before NOT NULL. + op.add_column( + "image_record", + sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True), + ) + # Pure set-based UPDATEs (no per-row params) — immune to the 65535 + # bind-parameter ceiling regardless of library size. + op.execute( + """ + UPDATE image_record AS ir + SET effective_date = COALESCE(p.post_date, ir.created_at) + FROM post AS p + WHERE ir.primary_post_id = p.id + """ + ) + op.execute( + """ + UPDATE image_record + SET effective_date = created_at + WHERE effective_date IS NULL + """ + ) + op.alter_column( + "image_record", + "effective_date", + nullable=False, + server_default=sa.text("now()"), + ) + # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC + # so the scroll is a forward index scan; raw SQL because alembic's + # column list doesn't express per-column DESC cleanly. + op.execute( + "CREATE INDEX ix_image_record_effective_date " + "ON image_record (effective_date DESC, id DESC)" + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_effective_date", table_name="image_record") + op.drop_column("image_record", "effective_date") diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 41b561c..1533532 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -8,26 +8,42 @@ from ..services.gallery_service import GalleryService gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") +def _parse_filters(): + """Parse the composable gallery filters from query args. Raises + ValueError (→ 400) on malformed ids. `tag_id` accepts a single id or a + comma-separated list (AND); `media` is image|video; `sort` is + newest|oldest.""" + tag_raw = request.args.get("tag_id") + tag_ids = ( + [int(x) for x in tag_raw.split(",") if x.strip()] if tag_raw else None + ) or None + post_id_raw = request.args.get("post_id") + post_id = int(post_id_raw) if post_id_raw else None + artist_id_raw = request.args.get("artist_id") + artist_id = int(artist_id_raw) if artist_id_raw else None + media = request.args.get("media") + media_type = media if media in ("image", "video") else None + sort = request.args.get("sort") + sort = sort if sort in ("newest", "oldest") else "newest" + return tag_ids, post_id, artist_id, media_type, sort + + @gallery_bp.route("/scroll", methods=["GET"]) async def scroll(): cursor = request.args.get("cursor") or None try: limit = int(request.args.get("limit", "50")) + tag_ids, post_id, artist_id, media_type, sort = _parse_filters() except ValueError: - return jsonify({"error": "limit must be an integer"}), 400 - tag_id_raw = request.args.get("tag_id") - tag_id = int(tag_id_raw) if tag_id_raw else None - post_id_raw = request.args.get("post_id") - post_id = int(post_id_raw) if post_id_raw else None - artist_id_raw = request.args.get("artist_id") - artist_id = int(artist_id_raw) if artist_id_raw else None + return jsonify({"error": "invalid filter or limit parameter"}), 400 async with get_session() as session: svc = GalleryService(session) try: page = await svc.scroll( - cursor=cursor, limit=limit, tag_id=tag_id, + cursor=cursor, limit=limit, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, + media_type=media_type, sort=sort, ) except ValueError as exc: return jsonify({"error": str(exc)}), 400 @@ -58,17 +74,16 @@ async def scroll(): @gallery_bp.route("/timeline", methods=["GET"]) async def timeline(): - tag_id_raw = request.args.get("tag_id") - tag_id = int(tag_id_raw) if tag_id_raw else None - post_id_raw = request.args.get("post_id") - post_id = int(post_id_raw) if post_id_raw else None - artist_id_raw = request.args.get("artist_id") - artist_id = int(artist_id_raw) if artist_id_raw else None + try: + tag_ids, post_id, artist_id, media_type, _sort = _parse_filters() + except ValueError: + return jsonify({"error": "invalid filter parameter"}), 400 async with get_session() as session: svc = GalleryService(session) try: buckets = await svc.timeline( - tag_id=tag_id, post_id=post_id, artist_id=artist_id + tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, + media_type=media_type, ) except ValueError as exc: return jsonify({"error": str(exc)}), 400 @@ -82,20 +97,16 @@ async def jump(): try: year = int(request.args["year"]) month = int(request.args["month"]) + tag_ids, post_id, artist_id, media_type, sort = _parse_filters() except (KeyError, ValueError): return jsonify({"error": "year and month query params required"}), 400 - tag_id_raw = request.args.get("tag_id") - tag_id = int(tag_id_raw) if tag_id_raw else None - post_id_raw = request.args.get("post_id") - post_id = int(post_id_raw) if post_id_raw else None - artist_id_raw = request.args.get("artist_id") - artist_id = int(artist_id_raw) if artist_id_raw else None async with get_session() as session: svc = GalleryService(session) try: cursor = await svc.jump_cursor( - year=year, month=month, tag_id=tag_id, + year=year, month=month, tag_ids=tag_ids, post_id=post_id, artist_id=artist_id, + media_type=media_type, sort=sort, ) except ValueError as exc: return jsonify({"error": str(exc)}), 400 diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 9061985..94bddff 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -194,15 +194,46 @@ async def remove_tag_from_image(image_id: int, tag_id: int): return "", 204 +@tags_bp.route("/tags/", methods=["GET"]) +async def get_tag(tag_id: int): + """Resolve a single tag (used by the gallery to label its active + tag-filter chip).""" + async with get_session() as session: + tag = await session.get(Tag, tag_id) + if tag is None: + return jsonify({"error": "tag not found"}), 404 + return jsonify( + { + "id": tag.id, + "name": tag.name, + "kind": tag.kind.value, + "fandom_id": tag.fandom_id, + } + ) + + @tags_bp.route("/tags/", methods=["PATCH"]) -async def rename_tag(tag_id: int): - body = await request.get_json() - if not body or "name" not in body: - return jsonify({"error": "name required"}), 400 +async def update_tag(tag_id: int): + """Rename and/or re-fandom a tag. Body may carry `name` and/or + `fandom_id` (a fandom tag id, or null to clear — character tags only). + `merge: true` resolves a collision by merging into the existing tag. + """ + body = await request.get_json() or {} + has_name = "name" in body + has_fandom = "fandom_id" in body + if not has_name and not has_fandom: + return jsonify({"error": "name or fandom_id required"}), 400 + do_merge = bool(body.get("merge")) async with get_session() as session: svc = TagService(session) try: - tag = await svc.rename(tag_id, body["name"]) + tag = None + if has_name: + tag = await svc.rename(tag_id, body["name"]) + if has_fandom: + tag = await svc.set_fandom( + tag_id, body["fandom_id"], merge=do_merge + ) except TagMergeConflict as exc: return jsonify( { @@ -219,7 +250,12 @@ async def rename_tag(tag_id: int): return jsonify({"error": str(exc)}), 400 await session.commit() return jsonify( - {"id": tag.id, "name": tag.name, "kind": tag.kind.value} + { + "id": tag.id, + "name": tag.name, + "kind": tag.kind.value, + "fandom_id": tag.fandom_id, + } ) diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index f43fd60..812a9fa 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -74,6 +74,17 @@ class ImageRecord(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + # Denormalized gallery sort key = COALESCE(primary post's post_date, + # created_at) (alembic 0035). The gallery used to compute this as a + # COALESCE across the Post outer join on every /scroll, which can't use + # an index and re-sorted a large slice of the library per page (×10 with + # the old serial batching). Materializing it lets the cursor scroll read + # ix_image_record_effective_date directly. Maintained by the importer + # (services/importer.py _apply_sidecar) when a primary post with a date + # is linked; plain inserts keep the created_at-equivalent server default. + effective_date: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 3331510..7690875 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -43,16 +43,17 @@ def decode_cursor(cursor: str) -> tuple[datetime, int]: def _effective_date_col(): - """SQL expression: COALESCE(post.post_date, image_record.created_at). + """The materialized gallery sort key: image_record.effective_date + (alembic 0035) = COALESCE(primary post's post_date, created_at), + maintained at write time by the importer. - Used as the canonical sort/group/filter key across the gallery so - images backfilled with primary_post_id (e.g. via tag_apply phase 4) - surface at their original publish date, not their FC import date. - Images without a Post (or with Post.post_date NULL) fall back to - image_record.created_at and still order coherently against - post-attached ones. + Canonical sort/group/filter key across the gallery so images attached + to a post surface at their original publish date, not their FC import + date — and, now that it's a single indexed column rather than a + COALESCE across the Post outer join, the cursor scroll is an index + range scan instead of a full re-sort per page. """ - return func.coalesce(Post.post_date, ImageRecord.created_at) + return ImageRecord.effective_date def _outer_join_primary_post(stmt: Select) -> Select: @@ -117,13 +118,43 @@ def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str return f"/images/thumbs/{bucket}/{sha256_hex}{ext}" -def _require_single_filter(tag_id, post_id, artist_id) -> None: - if sum(x is not None for x in (tag_id, post_id, artist_id)) > 1: +def _require_single_filter(tag_ids, post_id, artist_id) -> None: + """post_id is the post-detail view — it can't combine with the + composable filters. tag_ids + artist_id (+ media_type) compose freely + (AND).""" + if post_id is not None and (tag_ids or artist_id is not None): raise ValueError( - "tag_id, post_id, artist_id are mutually exclusive" + "post_id cannot be combined with tag or artist filters" ) +def _apply_scope(stmt, *, tag_ids, post_id, artist_id, media_type): + """Apply the composable gallery filters to a statement already joined + to Post via _outer_join_primary_post. + + - tag_ids: image must carry ALL of them — one correlated EXISTS per tag + (AND), which avoids the row-multiplication a multi-join would cause. + - post_id / artist_id: provenance EXISTS (post_id is exclusive, guarded + by _require_single_filter). + - media_type: 'image' | 'video' narrows by mime prefix. + """ + for tid in tag_ids or []: + stmt = stmt.where( + exists().where( + image_tag.c.image_record_id == ImageRecord.id, + image_tag.c.tag_id == tid, + ) + ) + prov = _provenance_clause(post_id, artist_id) + if prov is not None: + stmt = stmt.where(prov) + if media_type == "image": + stmt = stmt.where(ImageRecord.mime.like("image/%")) + elif media_type == "video": + stmt = stmt.where(ImageRecord.mime.like("video/%")) + return stmt + + def _provenance_clause(post_id, artist_id): """Correlated EXISTS clause (NOT a join) so an image with multiple matching provenance rows is returned exactly once and the @@ -179,35 +210,43 @@ class GalleryService: self, cursor: str | None, limit: int = 50, - tag_id: int | None = None, + tag_ids: list[int] | None = None, post_id: int | None = None, artist_id: int | None = None, + media_type: str | None = None, + sort: str = "newest", ) -> GalleryPage: if limit < 1 or limit > 200: raise ValueError("limit must be between 1 and 200") - _require_single_filter(tag_id, post_id, artist_id) + _require_single_filter(tag_ids, post_id, artist_id) eff = _effective_date_col() stmt = select(ImageRecord, Post.post_date, eff.label("eff")) stmt = _outer_join_primary_post(stmt) - if tag_id is not None: - stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( - image_tag.c.tag_id == tag_id - ) - prov = _provenance_clause(post_id, artist_id) - if prov is not None: - stmt = stmt.where(prov) + stmt = _apply_scope( + stmt, tag_ids=tag_ids, post_id=post_id, + artist_id=artist_id, media_type=media_type, + ) + descending = sort != "oldest" if cursor: cur_ts, cur_id = decode_cursor(cursor) - stmt = stmt.where( - or_( - eff < cur_ts, - and_(eff == cur_ts, ImageRecord.id < cur_id), + # The cursor is just (last eff, last id); the request's sort + # decides which side of it the next page lies on. + if descending: + stmt = stmt.where( + or_(eff < cur_ts, and_(eff == cur_ts, ImageRecord.id < cur_id)) + ) + else: + stmt = stmt.where( + or_(eff > cur_ts, and_(eff == cur_ts, ImageRecord.id > cur_id)) ) - ) - stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(limit + 1) + if descending: + stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()) + else: + stmt = stmt.order_by(eff.asc(), ImageRecord.id.asc()) + stmt = stmt.limit(limit + 1) rows = (await self.session.execute(stmt)).all() next_cursor = None @@ -243,9 +282,10 @@ class GalleryService: async def timeline( self, - tag_id: int | None = None, + tag_ids: list[int] | None = None, post_id: int | None = None, artist_id: int | None = None, + media_type: str | None = None, ) -> list[TimelineBucket]: eff = _effective_date_col() year_col = func.date_part("year", eff).label("yr") @@ -254,25 +294,23 @@ class GalleryService: year_col, month_col, func.count(ImageRecord.id).label("cnt") ) stmt = _outer_join_primary_post(stmt) - _require_single_filter(tag_id, post_id, artist_id) - if tag_id is not None: - stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( - image_tag.c.tag_id == tag_id - ) - prov = _provenance_clause(post_id, artist_id) - if prov is not None: - stmt = stmt.where(prov) + _require_single_filter(tag_ids, post_id, artist_id) + stmt = _apply_scope( + stmt, tag_ids=tag_ids, post_id=post_id, + artist_id=artist_id, media_type=media_type, + ) stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc()) rows = (await self.session.execute(stmt)).all() return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows] async def jump_cursor( - self, year: int, month: int, tag_id: int | None = None, + self, year: int, month: int, tag_ids: list[int] | None = None, post_id: int | None = None, artist_id: int | None = None, + media_type: str | None = None, sort: str = "newest", ) -> str | None: - """Returns a cursor that, when passed to scroll(), positions at the - first image of the given year-month (by effective_date, not - created_at). None if the bucket is empty. + """Returns a cursor that, when passed to scroll() with the same sort, + positions at the first image of the given year-month. None if the + bucket is empty. """ from sqlalchemy import extract @@ -282,22 +320,24 @@ class GalleryService: extract("month", eff) == month, ) stmt = _outer_join_primary_post(stmt) - _require_single_filter(tag_id, post_id, artist_id) - if tag_id is not None: - stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( - image_tag.c.tag_id == tag_id - ) - prov = _provenance_clause(post_id, artist_id) - if prov is not None: - stmt = stmt.where(prov) - stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()).limit(1) - first = (await self.session.execute(stmt)).first() + _require_single_filter(tag_ids, post_id, artist_id) + stmt = _apply_scope( + stmt, tag_ids=tag_ids, post_id=post_id, + artist_id=artist_id, media_type=media_type, + ) + descending = sort != "oldest" + if descending: + stmt = stmt.order_by(eff.desc(), ImageRecord.id.desc()) + else: + stmt = stmt.order_by(eff.asc(), ImageRecord.id.asc()) + first = (await self.session.execute(stmt.limit(1))).first() if first is None: return None record, eff_date = first - # Cursor is exclusive; we encode a cursor with id+1 so the row itself - # is the first result in the next scroll(). - return encode_cursor(eff_date, record.id + 1) + # Cursor is exclusive; nudge the id one past the boundary row (in the + # scan direction) so the row itself is the first result of scroll(). + boundary = record.id + 1 if descending else record.id - 1 + return encode_cursor(eff_date, boundary) async def get_image_with_tags(self, image_id: int) -> dict | None: record = await self.session.get(ImageRecord, image_id) @@ -357,17 +397,10 @@ class GalleryService: } async def _neighbors(self, record: ImageRecord) -> dict: - # Compute the boundary image's effective_date in Python (one query - # below + the SELECT we already have on `record`) and use it for - # the neighbor comparison. Cheaper than re-deriving in SQL via - # correlated subquery. - boundary_eff = record.created_at - if record.primary_post_id is not None: - post_date = (await self.session.execute( - select(Post.post_date).where(Post.id == record.primary_post_id) - )).scalar_one_or_none() - if post_date is not None: - boundary_eff = post_date + # The boundary image's sort key is materialized on the row now + # (alembic 0035) — read it directly instead of re-deriving COALESCE + # via an extra Post lookup. + boundary_eff = record.effective_date eff = _effective_date_col() prev_stmt = _outer_join_primary_post( diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 72713c5..213b86e 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -960,6 +960,14 @@ class Importer: sp.rollback() if record.primary_post_id is None: record.primary_post_id = post.id + # Keep the denormalized gallery sort key (alembic 0035) aligned with + # the primary post's publish date so /scroll orders off + # ix_image_record_effective_date instead of COALESCE-ing across the + # post join. Only override when THIS post is the primary AND carries + # a date; otherwise the column keeps its created_at-equivalent server + # default (matches the old COALESCE(post_date, created_at) fallback). + if record.primary_post_id == post.id and post.post_date is not None: + record.effective_date = post.post_date self.session.flush() def _copy_to_library( diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index efb0add..9f7a0e6 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -280,6 +280,69 @@ class TagService: await self.session.flush() return tag + async def set_fandom( + self, tag_id: int, fandom_id: int | None, *, merge: bool = False + ) -> Tag: + """Set / change / clear a character tag's fandom. + + Raises TagValidationError unless the tag is a character and fandom_id + (when given) references a fandom tag. If the change would collide with + an existing character of the same name in the TARGET fandom, raises + TagMergeConflict (the API turns that into a 409 merge hint) — unless + merge=True, in which case this tag is merged INTO that existing + character (a deliberate cross-fandom merge) and the surviving target + is returned. Passing fandom_id=None clears the fandom. + """ + tag = await self.session.get(Tag, tag_id) + if tag is None: + raise TagValidationError(f"Tag {tag_id} not found") + if tag.kind != TagKind.character: + raise TagValidationError("Only character tags can have a fandom") + if fandom_id is not None: + fandom = await self.session.get(Tag, fandom_id) + if fandom is None or fandom.kind != TagKind.fandom: + raise TagValidationError( + f"fandom_id {fandom_id} does not reference a fandom tag" + ) + if fandom_id == tag.fandom_id: + return tag + + # Collision: another character with the same name already lives in the + # target fandom. Mirrors rename's (name, kind, fandom_id) uniqueness. + clash_stmt = ( + select(Tag) + .where(Tag.name == tag.name) + .where(Tag.kind == TagKind.character) + .where( + Tag.fandom_id.is_(None) + if fandom_id is None + else Tag.fandom_id == fandom_id + ) + .where(Tag.id != tag_id) + ) + clash = (await self.session.execute(clash_stmt)).scalar_one_or_none() + if clash is not None: + if not merge: + source_image_count = await self.session.scalar( + select(func.count()) + .select_from(image_tag) + .where(image_tag.c.tag_id == tag_id) + ) + will_alias = await self._keep_as_alias(tag_id) + raise TagMergeConflict( + f"A character named {tag.name!r} already exists in that fandom", + target_id=clash.id, + target_name=clash.name, + source_image_count=int(source_image_count or 0), + will_alias=will_alias, + ) + await self._do_merge(tag, clash) + return clash + + tag.fandom_id = fandom_id + await self.session.flush() + return tag + async def merge(self, source_id: int, target_id: int) -> MergeResult: """Transactionally repoint every FK from source→target, optionally keep source's name as a tagger alias, delete source. Atomic: any @@ -298,7 +361,14 @@ class TagService: raise TagValidationError( "Tags must be the same kind and fandom to merge" ) + return await self._do_merge(source, target) + async def _do_merge(self, source: Tag, target: Tag) -> MergeResult: + """Repoint every FK source→target, optionally keep source's name as a + tagger alias, delete source. NO kind/fandom validation — callers that + need it (public merge()) validate first; set_fandom's collision + resolution calls this directly for a deliberate CROSS-fandom merge.""" + source_id, target_id = source.id, target.id keep_as_alias = await self._keep_as_alias(source_id) source_name = source.name source_kind = source.kind diff --git a/frontend/src/components/discovery/TagCard.vue b/frontend/src/components/discovery/TagCard.vue index 9eb27ad..db7b3df 100644 --- a/frontend/src/components/discovery/TagCard.vue +++ b/frontend/src/components/discovery/TagCard.vue @@ -55,6 +55,12 @@ /> + +
+ + + + +
+ {{ store.tagLabels[id] || `#${id}` }} + {{ store.artistLabel || `Artist #${store.filter.artist_id}` }} +
+ + + + + All + Images + Videos + + + + + Clear +
+ + + + + diff --git a/frontend/src/components/gallery/GalleryItem.vue b/frontend/src/components/gallery/GalleryItem.vue index 8d9ba13..cab8b9e 100644 --- a/frontend/src/components/gallery/GalleryItem.vue +++ b/frontend/src/components/gallery/GalleryItem.vue @@ -9,11 +9,16 @@ >