From a92817677d88d18f642991bac8ed3d418531db95 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 12:34:28 -0400 Subject: [PATCH 01/10] fix(router): reset tab title on navigation (artist name stuck on other tabs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArtistView set document.title to " — FabledCurator" on load but nothing reset it when navigating away, so the artist name stuck on the Showcase/Gallery tab title (operator-flagged 2026-06-07). Add a router.afterEach that sets the title from meta.title on every navigation; detail views with no meta.title reset to the default and then set their own dynamic title. Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend/src/router.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/frontend/src/router.js b/frontend/src/router.js index f3b6cbe..9a6a05d 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -54,4 +54,19 @@ const router = createRouter({ routes }) +const DEFAULT_TITLE = 'FabledCurator' + +// Keep the tab title in sync with the route on EVERY navigation. List routes set +// it from meta.title; detail routes (artist, series) have no meta.title, so they +// reset to the default here and then overwrite it with their own dynamic title +// (e.g. ArtistView sets " — FabledCurator"). Without this reset the +// previous view's dynamic title stuck around — an artist name showing on the +// Showcase tab, operator-flagged 2026-06-07. +router.afterEach((to) => { + if (typeof document === 'undefined') return + document.title = to.meta?.title + ? `${to.meta.title} — ${DEFAULT_TITLE}` + : DEFAULT_TITLE +}) + export default router -- 2.52.0 From 677317244e47b2f84dd09dd6b383047e66537a24 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 12:38:33 -0400 Subject: [PATCH 02/10] fix(modal): Enter accepts the fandom instead of reopening the dropdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Enter handler listened in the bubbling phase, so Vuetify's own input handler (which opens the menu on Enter) fired first and my accept logic saw the menu already opening and bailed — Enter popped the dropdown instead of submitting. Bind it in the capture phase so it runs first, and stop the event when a fandom is already selected so Vuetify never reopens the menu (operator-flagged 2026-06-07). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/components/modal/FandomPicker.vue | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/modal/FandomPicker.vue b/frontend/src/components/modal/FandomPicker.vue index 8f1147e..cb84aa2 100644 --- a/frontend/src/components/modal/FandomPicker.vue +++ b/frontend/src/components/modal/FandomPicker.vue @@ -19,7 +19,7 @@ :item-title="(f) => f.name" :item-value="(f) => f.id" label="Fandom" clearable density="compact" - @keydown.enter="onSearchEnter" + @keydown.enter.capture="onSearchEnter" @keydown.tab="onSearchTab" /> @@ -92,12 +92,20 @@ function onConfirm () { if (f) emit('confirm', f) } -// Enter on the search field. When the menu is open Vuetify is picking the -// highlighted item (it sets selectedId + closes the menu) — don't also accept -// on that keystroke. With the menu closed and a value chosen, Enter accepts. -function onSearchEnter () { +// Enter on the search field — bound in the CAPTURE phase so this runs BEFORE +// Vuetify's own input keydown handler (which opens the menu on Enter). When the +// menu is open, let Vuetify pick the highlighted item (it sets selectedId + +// closes the menu). When it's closed and a fandom is already chosen, accept it +// and stop the event so Vuetify never (re)opens the menu — that re-open was the +// bug: Enter popped the dropdown instead of submitting (operator-flagged +// 2026-06-07). +function onSearchEnter (e) { if (menuOpen.value) return - if (selectedId.value != null) onConfirm() + if (selectedId.value != null) { + e.preventDefault() + e.stopPropagation() + onConfirm() + } } // Tab from the search field goes to the "new fandom" text field (operator- -- 2.52.0 From 43b02d79a4c11d2c72bc81aac1d36521c1f67538 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 16:22:09 -0400 Subject: [PATCH 03/10] fix(infra): size Postgres /dev/shm via tmpfs mount (shm_size ignored under Swarm) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prod stack runs under Docker Swarm (docker stack deploy), which SILENTLY IGNORES `shm_size` — container inspect showed ShmSize still 64MB after the a183be7 fix, and vacuum_analyze kept hitting DiskFull resizing a ~64MB POSIX DSM segment in /dev/shm (operator-flagged 2026-06-07). Replace the ignored `shm_size: 512m` with a tmpfs mount on /dev/shm (size 512MB), which Swarm AND plain Compose both honor. Requires a stack redeploy to take effect. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f1088ef..ee793f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,17 +21,23 @@ services: postgres: image: pgvector/pgvector:pg16 - # Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) and parallel queries - # allocate larger shared-memory segments and fail with - # "could not resize shared memory segment ... No space left on device" - # (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB). - shm_size: 512m environment: POSTGRES_USER: ${DB_USER:-fabledcurator} POSTGRES_PASSWORD: ${DB_PASSWORD:-fabledcurator_dev} POSTGRES_DB: ${DB_NAME:-fabledcurator} volumes: - postgres_data:/var/lib/postgresql/data + # Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) + parallel queries + # allocate a POSIX dynamic-shared-memory segment in /dev/shm and fail with + # "could not resize shared memory segment ... No space left on device" + # (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB). + # NOTE: a `shm_size:` key is SILENTLY IGNORED under Docker Swarm + # (`docker stack deploy` — the prod deployment here), so size /dev/shm via + # a tmpfs mount instead — honored by both Swarm and plain Compose. + - type: tmpfs + target: /dev/shm + tmpfs: + size: 536870912 # 512 MB healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-fabledcurator}"] interval: 10s -- 2.52.0 From 1804a2c6221a104c0573ca7411791e7595f9254b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 16:31:55 -0400 Subject: [PATCH 04/10] =?UTF-8?q?feat(series):=20chapter=20layer=20over=20?= =?UTF-8?q?series=5Fpage=20=E2=80=94=20backend=20(FC-6.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an ordered chapter layer to series. Reading order becomes (series_chapter.chapter_number, series_page.page_number); a chapter may be a placeholder reserving a slot, and carries an optional parsed stated-page range used to flag missing-page gaps. An image still lives in at most one series ⇒ one chapter (image_id stays UNIQUE). - models: series_chapter; series_page gains chapter_id (NOT NULL, cascade) + stated_page. Migration 0040 backfills every existing series into one auto-chapter holding its current flat pages — no data loss. - SeriesService: chapter CRUD (create/update/reorder/delete/merge), page→chapter assignment, reorder_pages, chapter-aware set_cover; list_pages now returns chapters[] + gaps[] alongside a back-compat flat pages[]. Legacy series-wide reorder operates on the single default chapter and rejects multi-chapter series. - API: chapter endpoints under /api/series//chapters; POST pages accepts an optional chapter_id. - TagService.merge now repoints series_chapter too, so a merged series' chapters (and their pages) survive the source tag's deletion instead of cascading away. - Tests: new chapter suite; updated the 4 direct SeriesPage(...) constructions to supply chapter_id. Frontend (chapter-aware manage view + reader) lands next; until then the existing UI keeps working via the flat pages[] + single default chapter. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0040_series_chapters.py | 108 +++++++ backend/app/api/tags.py | 160 +++++++++- backend/app/models/__init__.py | 2 + backend/app/models/series_chapter.py | 47 +++ backend/app/models/series_page.py | 17 +- backend/app/services/series_service.py | 376 +++++++++++++++++++++-- backend/app/services/tag_service.py | 10 + tests/test_cleanup_service.py | 8 +- tests/test_series_chapters.py | 193 ++++++++++++ tests/test_series_model.py | 18 +- tests/test_tag_merge.py | 13 +- 11 files changed, 909 insertions(+), 43 deletions(-) create mode 100644 alembic/versions/0040_series_chapters.py create mode 100644 backend/app/models/series_chapter.py create mode 100644 tests/test_series_chapters.py diff --git a/alembic/versions/0040_series_chapters.py b/alembic/versions/0040_series_chapters.py new file mode 100644 index 0000000..a0808df --- /dev/null +++ b/alembic/versions/0040_series_chapters.py @@ -0,0 +1,108 @@ +"""series chapters: chapter layer over series_page (FC-6.1) + +Revision ID: 0040 +Revises: 0039 +Create Date: 2026-06-07 + +A series (Tag kind='series') gains an ordered chapter layer. Reading order +becomes (series_chapter.chapter_number, series_page.page_number). Every existing +series is backfilled into a single auto-chapter (chapter_number=1) holding its +current flat pages, so no data is lost and the old flat ordering is preserved. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0040" +down_revision: Union[str, None] = "0039" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_chapter", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("chapter_number", sa.Integer, nullable=False), + sa.Column("title", sa.Text, nullable=True), + sa.Column( + "is_placeholder", sa.Boolean, nullable=False, server_default="false" + ), + sa.Column("stated_page_start", sa.Integer, nullable=True), + sa.Column("stated_page_end", sa.Integer, nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + ) + op.create_index( + "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"] + ) + + # New columns on series_page; chapter_id starts nullable so we can backfill. + op.add_column( + "series_page", sa.Column("chapter_id", sa.Integer, nullable=True) + ) + op.add_column( + "series_page", sa.Column("stated_page", sa.Integer, nullable=True) + ) + + conn = op.get_bind() + # One auto-chapter per existing series (any series_tag_id present in pages). + conn.execute( + sa.text( + "INSERT INTO series_chapter " + "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) " + "SELECT DISTINCT series_tag_id, 1, false, now(), now() " + "FROM series_page" + ) + ) + # Point every existing page at its series' auto-chapter. + conn.execute( + sa.text( + "UPDATE series_page sp " + "SET chapter_id = sc.id " + "FROM series_chapter sc " + "WHERE sc.series_tag_id = sp.series_tag_id" + ) + ) + + # Now lock chapter_id down: NOT NULL + FK (cascade) + index. + op.alter_column("series_page", "chapter_id", nullable=False) + op.create_foreign_key( + "fk_series_page_chapter_id", + "series_page", + "series_chapter", + ["chapter_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + "ix_series_page_chapter_id", "series_page", ["chapter_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_series_page_chapter_id", table_name="series_page") + op.drop_constraint( + "fk_series_page_chapter_id", "series_page", type_="foreignkey" + ) + op.drop_column("series_page", "stated_page") + op.drop_column("series_page", "chapter_id") + op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter") + op.drop_table("series_chapter") diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 288ade6..d77dca2 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -368,6 +368,31 @@ def _series_err(exc: SeriesError): return jsonify({"error": msg}), status +def _opt_int(body, key: str): + """(value, error) — value is None when absent, error is (json, status).""" + if not body or body.get(key) is None: + return None, None + try: + return int(body[key]), None + except (TypeError, ValueError): + return None, (jsonify({"error": f"{key} must be an integer"}), 400) + + +def _parse_int_list(body, key: str, *, max_ids: int = 500): + """(list, error) for a required list of ints under `key`.""" + if not body or key not in body: + return None, (jsonify({"error": f"{key} required"}), 400) + raw = body[key] + if not isinstance(raw, list) or not raw: + return None, (jsonify({"error": f"{key} must be a non-empty list"}), 400) + if len(raw) > max_ids: + return None, (jsonify({"error": f"too many ids (max {max_ids})"}), 400) + try: + return [int(x) for x in raw], None + except (TypeError, ValueError): + return None, (jsonify({"error": f"{key} must be integers"}), 400) + + @tags_bp.route("/series//pages", methods=["GET"]) async def series_pages(tag_id: int): async with get_session() as session: @@ -384,9 +409,14 @@ async def series_add(tag_id: int): ids, err = _parse_bulk_ids(body, max_ids=500) if err: return err + chapter_id, cerr = _opt_int(body, "chapter_id") + if cerr: + return cerr async with get_session() as session: try: - n = await SeriesService(session).add_images(tag_id, ids) + n = await SeriesService(session).add_images( + tag_id, ids, chapter_id=chapter_id + ) except SeriesError as exc: return _series_err(exc) await session.commit() @@ -439,3 +469,131 @@ async def series_cover(tag_id: int): return _series_err(exc) await session.commit() return jsonify({"ok": True}) + + +# ---- chapters (FC-6.1) ---------------------------------------------------- + + +@tags_bp.route("/series//chapters", methods=["POST"]) +async def series_chapter_create(tag_id: int): + body = await request.get_json() or {} + title = body.get("title") + if title is not None and not isinstance(title, str): + return jsonify({"error": "title must be a string"}), 400 + is_placeholder = bool(body.get("is_placeholder", False)) + start, serr = _opt_int(body, "stated_page_start") + if serr: + return serr + end, eerr = _opt_int(body, "stated_page_end") + if eerr: + return eerr + async with get_session() as session: + try: + ch = await SeriesService(session).create_chapter( + tag_id, + title=title, + is_placeholder=is_placeholder, + stated_page_start=start, + stated_page_end=end, + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(ch) + + +@tags_bp.route("/series//chapters/reorder", methods=["POST"]) +async def series_chapter_reorder(tag_id: int): + body = await request.get_json() + ids, err = _parse_int_list(body, "chapter_ids") + if err: + return err + async with get_session() as session: + try: + await SeriesService(session).reorder_chapters(tag_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters/", methods=["PATCH"] +) +async def series_chapter_update(tag_id: int, chapter_id: int): + body = await request.get_json() or {} + kwargs: dict = {} + if "title" in body: + if body["title"] is not None and not isinstance(body["title"], str): + return jsonify({"error": "title must be a string"}), 400 + kwargs.update(set_title=True, title=body["title"]) + if "stated_page_start" in body: + start, serr = _opt_int(body, "stated_page_start") + if serr: + return serr + kwargs.update(set_start=True, stated_page_start=start) + if "stated_page_end" in body: + end, eerr = _opt_int(body, "stated_page_end") + if eerr: + return eerr + kwargs.update(set_end=True, stated_page_end=end) + async with get_session() as session: + try: + await SeriesService(session).update_chapter( + tag_id, chapter_id, **kwargs + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters/", methods=["DELETE"] +) +async def series_chapter_delete(tag_id: int, chapter_id: int): + async with get_session() as session: + try: + await SeriesService(session).delete_chapter(tag_id, chapter_id) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters//merge", methods=["POST"] +) +async def series_chapter_merge(tag_id: int, chapter_id: int): + body = await request.get_json() + target, terr = _opt_int(body, "target_chapter_id") + if terr: + return terr + if target is None: + return jsonify({"error": "target_chapter_id required"}), 400 + async with get_session() as session: + try: + moved = await SeriesService(session).merge_chapter( + tag_id, chapter_id, target + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"moved_count": moved}) + + +@tags_bp.route( + "/series//chapters//reorder", methods=["POST"] +) +async def series_chapter_reorder_pages(tag_id: int, chapter_id: int): + body = await request.get_json() + ids, err = _parse_bulk_ids(body, max_ids=500) + if err: + return err + async with get_session() as session: + try: + await SeriesService(session).reorder_pages(tag_id, chapter_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index b8854b1..bfde14b 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -18,6 +18,7 @@ from .patreon_failed_media import PatreonFailedMedia from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment +from .series_chapter import SeriesChapter from .series_page import SeriesPage from .source import Source from .tag import Tag, TagKind, image_tag @@ -39,6 +40,7 @@ __all__ = [ "PatreonSeenMedia", "Post", "PostAttachment", + "SeriesChapter", "SeriesPage", "ImageRecord", "ImageProvenance", diff --git a/backend/app/models/series_chapter.py b/backend/app/models/series_chapter.py new file mode 100644 index 0000000..4012174 --- /dev/null +++ b/backend/app/models/series_chapter.py @@ -0,0 +1,47 @@ +"""SeriesChapter — an ordered chapter/part within a series. + +A series IS a Tag(kind='series'); a chapter groups ordered SeriesPages under it. +Reading order is (chapter.chapter_number, series_page.page_number): chapter_number +sets the order of chapters, page_number orders pages within a chapter. + +chapter_number is an ordering key only (not unique) — reorder rewrites 1..N +wholesale, mirroring series_page.page_number, so a reorder can't transiently +collide on a unique index. + +A chapter may be a placeholder (is_placeholder=True) — a reserved empty slot for +a section the operator doesn't have yet; it holds no pages and shows as a gap in +the reader. stated_page_start/end carry the page range parsed from the source +post (FC-6.2), used to flag missing-page gaps; both are nullable when unknown. +""" + +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, Text, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesChapter(Base): + __tablename__ = "series_chapter" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + chapter_number: Mapped[int] = mapped_column(Integer, nullable=False) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + is_placeholder: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default="false" + ) + stated_page_start: Mapped[int | None] = mapped_column(Integer, nullable=True) + stated_page_end: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/models/series_page.py b/backend/app/models/series_page.py index 1ebcd7d..48384ef 100644 --- a/backend/app/models/series_page.py +++ b/backend/app/models/series_page.py @@ -1,9 +1,12 @@ """SeriesPage — ordered image membership for a series-kind Tag. -A series IS a Tag with kind='series'; series_page gives it ordered pages. -An image belongs to at most one series (UNIQUE image_id). Cover = the -lowest page_number. page_number is an ordering key only (not unique) — -reorder rewrites 1..N wholesale. +A series IS a Tag with kind='series'; series_page gives it ordered pages, +grouped into chapters (FC-6). An image belongs to at most one series +(UNIQUE image_id) ⇒ at most one chapter. Reading order is +(chapter.chapter_number, series_page.page_number): page_number orders pages +WITHIN a chapter and is an ordering key only (not unique) — reorder rewrites +1..N wholesale. stated_page carries the page number parsed from the source +post (FC-6.2), nullable when unknown. """ from datetime import datetime @@ -21,12 +24,18 @@ class SeriesPage(Base): series_tag_id: Mapped[int] = mapped_column( ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True ) + chapter_id: Mapped[int] = mapped_column( + ForeignKey("series_chapter.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) image_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, unique=True, ) page_number: Mapped[int] = mapped_column(Integer, nullable=False) + stated_page: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/services/series_service.py b/backend/app/services/series_service.py index 361caa5..4ff19e9 100644 --- a/backend/app/services/series_service.py +++ b/backend/app/services/series_service.py @@ -1,4 +1,8 @@ -"""Series = a series-kind Tag + ordered series_page membership. +"""Series = a series-kind Tag + ordered chapters, each holding ordered pages. + +Reading order is (series_chapter.chapter_number, series_page.page_number). A +chapter may be a placeholder (no pages) reserving a slot. An image lives in at +most one series ⇒ one chapter (series_page.image_id is UNIQUE). All mutations are Core/set-based and run in the request transaction. """ @@ -8,6 +12,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from ..models import ImageRecord, Tag, TagKind +from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage from .gallery_service import thumbnail_url @@ -21,6 +26,8 @@ class SeriesService: def __init__(self, session: AsyncSession): self.session = session + # ---- guards / helpers ------------------------------------------------- + async def _require_series(self, series_tag_id: int) -> Tag: tag = await self.session.get(Tag, series_tag_id) if tag is None: @@ -32,7 +39,7 @@ class SeriesService: @staticmethod def _clean_ids(ids: list[int]) -> list[int]: if not ids: - raise SeriesError("image_ids must be a non-empty list") + raise SeriesError("ids must be a non-empty list") if len(ids) > 500: raise SeriesError("selection too large (max 500)") seen: set[int] = set() @@ -43,51 +50,172 @@ class SeriesService: out.append(int(x)) return out - async def _member_order(self, series_tag_id: int) -> list[int]: + async def _require_chapter(self, series_tag_id: int, chapter_id: int): + row = ( + await self.session.execute( + select(SeriesChapter).where( + and_( + SeriesChapter.id == chapter_id, + SeriesChapter.series_tag_id == series_tag_id, + ) + ) + ) + ).scalar_one_or_none() + if row is None: + raise SeriesError( + f"chapter {chapter_id} is not in series {series_tag_id}" + ) + return row + + async def _ensure_default_chapter(self, series_tag_id: int) -> int: + """Lowest-numbered chapter of the series, creating a first chapter if + the series has none (a fresh series, or the legacy single-chapter path).""" + existing = await self.session.scalar( + select(SeriesChapter.id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by(SeriesChapter.chapter_number.asc()) + .limit(1) + ) + if existing is not None: + return existing + res = await self.session.execute( + pg_insert(SeriesChapter) + .values(series_tag_id=series_tag_id, chapter_number=1) + .returning(SeriesChapter.id) + ) + return res.scalar_one() + + async def _chapter_page_order(self, chapter_id: int) -> list[int]: rows = ( await self.session.execute( select(SeriesPage.image_id) - .where(SeriesPage.series_tag_id == series_tag_id) + .where(SeriesPage.chapter_id == chapter_id) .order_by(SeriesPage.page_number.asc()) ) ).scalars().all() return list(rows) + async def _chapter_order(self, series_tag_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(SeriesChapter.id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by(SeriesChapter.chapter_number.asc()) + ) + ).scalars().all() + return list(rows) + + # ---- read ------------------------------------------------------------- + + @staticmethod + def _gaps(chapters: list[dict]) -> list[dict]: + """Missing-page gaps between consecutive chapters with stated ranges.""" + out: list[dict] = [] + prev = None + for ch in chapters: + start = ch["stated_page_start"] + if ( + prev is not None + and prev["stated_page_end"] is not None + and start is not None + and start > prev["stated_page_end"] + 1 + ): + out.append( + { + "after_chapter_id": prev["id"], + "start": prev["stated_page_end"] + 1, + "end": start - 1, + } + ) + prev = ch + return out + async def list_pages(self, series_tag_id: int) -> dict: tag = await self._require_series(series_tag_id) rows = ( await self.session.execute( select( + SeriesChapter.id.label("chapter_id"), + SeriesChapter.chapter_number, + SeriesChapter.title, + SeriesChapter.is_placeholder, + SeriesChapter.stated_page_start, + SeriesChapter.stated_page_end, SeriesPage.image_id, SeriesPage.page_number, + SeriesPage.stated_page, ImageRecord.sha256, ImageRecord.mime, ImageRecord.path, ImageRecord.thumbnail_path, ) - .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) - .where(SeriesPage.series_tag_id == series_tag_id) - .order_by(SeriesPage.page_number.asc()) + .select_from(SeriesChapter) + .outerjoin(SeriesPage, SeriesPage.chapter_id == SeriesChapter.id) + .outerjoin(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by( + SeriesChapter.chapter_number.asc(), + SeriesPage.page_number.asc(), + ) ) ).all() + + chapters: list[dict] = [] + flat: list[dict] = [] + by_id: dict[int, dict] = {} + for r in rows: + ch = by_id.get(r.chapter_id) + if ch is None: + ch = { + "id": r.chapter_id, + "chapter_number": r.chapter_number, + "title": r.title, + "is_placeholder": r.is_placeholder, + "stated_page_start": r.stated_page_start, + "stated_page_end": r.stated_page_end, + "pages": [], + } + by_id[r.chapter_id] = ch + chapters.append(ch) + if r.image_id is None: + continue # placeholder / empty chapter + page = { + "image_id": r.image_id, + "chapter_id": r.chapter_id, + "page_number": r.page_number, + "stated_page": r.stated_page, + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), + "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", + } + ch["pages"].append(page) + flat.append(page) + return { "series": {"id": tag.id, "name": tag.name}, - "pages": [ - { - "image_id": r.image_id, - "page_number": r.page_number, - "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), - "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", - } - for r in rows - ], + "chapters": chapters, + "pages": flat, # back-compat: flat reading order across chapters + "gaps": self._gaps(chapters), } + # ---- pages ------------------------------------------------------------ + async def add_images( - self, series_tag_id: int, image_ids: list[int] + self, + series_tag_id: int, + image_ids: list[int], + chapter_id: int | None = None, + stated_pages: dict[int, int] | None = None, ) -> int: + """Append images as pages of a chapter (the series' default chapter when + chapter_id is None). Images already in THIS series are left untouched; + images in another series are moved here (image_id is UNIQUE).""" await self._require_series(series_tag_id) ids = self._clean_ids(image_ids) + if chapter_id is None: + chapter_id = await self._ensure_default_chapter(series_tag_id) + else: + await self._require_chapter(series_tag_id, chapter_id) + existing = dict( ( await self.session.execute( @@ -96,29 +224,30 @@ class SeriesService: ) ).all() ) - # Already in THIS series → leave untouched (no churn). to_add = [i for i in ids if existing.get(i) != series_tag_id] if not to_add: return 0 - # Remove any prior membership for the to_add images (the "move"). + # Move: drop any prior membership for these images (UNIQUE image_id). await self.session.execute( - SeriesPage.__table__.delete().where( - SeriesPage.image_id.in_(to_add) - ) + SeriesPage.__table__.delete().where(SeriesPage.image_id.in_(to_add)) ) max_pn = ( await self.session.scalar( - select(func.coalesce(func.max(SeriesPage.page_number), 0)) - .where(SeriesPage.series_tag_id == series_tag_id) + select(func.coalesce(func.max(SeriesPage.page_number), 0)).where( + SeriesPage.chapter_id == chapter_id + ) ) ) or 0 + sp = stated_pages or {} await self.session.execute( pg_insert(SeriesPage).values( [ { "series_tag_id": series_tag_id, + "chapter_id": chapter_id, "image_id": iid, "page_number": max_pn + offset, + "stated_page": sp.get(iid), } for offset, iid in enumerate(to_add, start=1) ] @@ -141,32 +270,211 @@ class SeriesService: ) return res.rowcount or 0 - async def reorder( - self, series_tag_id: int, ordered_image_ids: list[int] + async def reorder_pages( + self, series_tag_id: int, chapter_id: int, ordered_image_ids: list[int] ) -> None: await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) ordered = self._clean_ids(ordered_image_ids) - current = set(await self._member_order(series_tag_id)) + current = set(await self._chapter_page_order(chapter_id)) if set(ordered) != current or len(ordered) != len(current): raise SeriesError( - "ordered image_ids must exactly match series membership" + "ordered image_ids must exactly match the chapter's pages" ) for idx, iid in enumerate(ordered, start=1): await self.session.execute( update(SeriesPage) .where( and_( - SeriesPage.series_tag_id == series_tag_id, + SeriesPage.chapter_id == chapter_id, SeriesPage.image_id == iid, ) ) .values(page_number=idx) ) - async def set_cover(self, series_tag_id: int, image_id: int) -> None: + async def reorder( + self, series_tag_id: int, ordered_image_ids: list[int] + ) -> None: + """Legacy series-wide reorder — reorders the default chapter. Valid only + for single-chapter series (the only shape the pre-chapter UI produced); + multi-chapter series must use reorder_pages with an explicit chapter.""" await self._require_series(series_tag_id) - order = await self._member_order(series_tag_id) - if image_id not in order: + chapters = await self._chapter_order(series_tag_id) + if len(chapters) > 1: + raise SeriesError( + "series has multiple chapters; reorder a specific chapter" + ) + chapter_id = ( + chapters[0] + if chapters + else await self._ensure_default_chapter(series_tag_id) + ) + await self.reorder_pages(series_tag_id, chapter_id, ordered_image_ids) + + # ---- chapters --------------------------------------------------------- + + async def create_chapter( + self, + series_tag_id: int, + *, + title: str | None = None, + is_placeholder: bool = False, + stated_page_start: int | None = None, + stated_page_end: int | None = None, + ) -> dict: + await self._require_series(series_tag_id) + max_cn = ( + await self.session.scalar( + select( + func.coalesce(func.max(SeriesChapter.chapter_number), 0) + ).where(SeriesChapter.series_tag_id == series_tag_id) + ) + ) or 0 + res = await self.session.execute( + pg_insert(SeriesChapter) + .values( + series_tag_id=series_tag_id, + chapter_number=max_cn + 1, + title=title, + is_placeholder=is_placeholder, + stated_page_start=stated_page_start, + stated_page_end=stated_page_end, + ) + .returning(SeriesChapter.id, SeriesChapter.chapter_number) + ) + row = res.one() + return {"id": row.id, "chapter_number": row.chapter_number} + + async def update_chapter( + self, + series_tag_id: int, + chapter_id: int, + *, + title: str | None = None, + stated_page_start: int | None = None, + stated_page_end: int | None = None, + set_title: bool = False, + set_start: bool = False, + set_end: bool = False, + ) -> None: + """Partial chapter edit. The set_* flags say which fields to write (so + None can be written explicitly, e.g. clearing a stated page).""" + await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) + values: dict = {} + if set_title: + values["title"] = title + if set_start: + values["stated_page_start"] = stated_page_start + if set_end: + values["stated_page_end"] = stated_page_end + if not values: + return + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == chapter_id) + .values(**values) + ) + + async def _renumber_chapters(self, series_tag_id: int) -> None: + for idx, cid in enumerate( + await self._chapter_order(series_tag_id), start=1 + ): + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == cid) + .values(chapter_number=idx) + ) + + async def reorder_chapters( + self, series_tag_id: int, ordered_chapter_ids: list[int] + ) -> None: + await self._require_series(series_tag_id) + ordered = self._clean_ids(ordered_chapter_ids) + current = set(await self._chapter_order(series_tag_id)) + if set(ordered) != current or len(ordered) != len(current): + raise SeriesError( + "ordered chapter_ids must exactly match the series' chapters" + ) + for idx, cid in enumerate(ordered, start=1): + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == cid) + .values(chapter_number=idx) + ) + + async def delete_chapter(self, series_tag_id: int, chapter_id: int) -> None: + await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) + # Pages cascade-delete with the chapter (FK ondelete=CASCADE). + await self.session.execute( + SeriesChapter.__table__.delete().where( + SeriesChapter.id == chapter_id + ) + ) + await self._renumber_chapters(series_tag_id) + + async def merge_chapter( + self, series_tag_id: int, source_chapter_id: int, target_chapter_id: int + ) -> int: + """Move source chapter's pages onto the end of the target chapter, then + delete the now-empty source chapter. Returns the number of pages moved.""" + await self._require_series(series_tag_id) + if source_chapter_id == target_chapter_id: + raise SeriesError("cannot merge a chapter into itself") + await self._require_chapter(series_tag_id, source_chapter_id) + await self._require_chapter(series_tag_id, target_chapter_id) + moving = await self._chapter_page_order(source_chapter_id) + max_pn = ( + await self.session.scalar( + select(func.coalesce(func.max(SeriesPage.page_number), 0)).where( + SeriesPage.chapter_id == target_chapter_id + ) + ) + ) or 0 + for offset, iid in enumerate(moving, start=1): + await self.session.execute( + update(SeriesPage) + .where( + and_( + SeriesPage.chapter_id == source_chapter_id, + SeriesPage.image_id == iid, + ) + ) + .values(chapter_id=target_chapter_id, page_number=max_pn + offset) + ) + await self.session.execute( + SeriesChapter.__table__.delete().where( + SeriesChapter.id == source_chapter_id + ) + ) + await self._renumber_chapters(series_tag_id) + return len(moving) + + async def set_cover(self, series_tag_id: int, image_id: int) -> None: + """Cover = the (chapter_number=1, page_number=1) image. Bring the image's + chapter to the front and the image to the front of its chapter.""" + await self._require_series(series_tag_id) + row = ( + await self.session.execute( + select(SeriesPage.chapter_id) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + SeriesPage.image_id == image_id, + ) + ) + ) + ).scalar_one_or_none() + if row is None: raise SeriesError(f"image {image_id} is not in this series") - new_order = [image_id] + [i for i in order if i != image_id] - await self.reorder(series_tag_id, new_order) + chapter_id = row + chapters = await self._chapter_order(series_tag_id) + if chapters and chapters[0] != chapter_id: + new_chapters = [chapter_id] + [c for c in chapters if c != chapter_id] + await self.reorder_chapters(series_tag_id, new_chapters) + pages = await self._chapter_page_order(chapter_id) + if pages and pages[0] != image_id: + new_pages = [image_id] + [p for p in pages if p != image_id] + await self.reorder_pages(series_tag_id, chapter_id, new_pages) diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index 255311e..c798bfb 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -515,8 +515,18 @@ class TagService: ) async def _repoint_series_pages(self, src: int, tgt: int) -> None: + from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage + # Move the chapters first so the pages' chapter_id FK stays valid: a + # chapter left pointing at src would cascade-delete (with its pages) when + # src is removed. chapter_number may now collide across the merged set — + # acceptable (it's an ordering key, not unique). + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.series_tag_id == src) + .values(series_tag_id=tgt) + ) # image_id is UNIQUE across series_page and src != tgt, so an # image in src's series cannot already be in tgt's — no collision. await self.session.execute( diff --git a/tests/test_cleanup_service.py b/tests/test_cleanup_service.py index fc5db17..ba6242f 100644 --- a/tests/test_cleanup_service.py +++ b/tests/test_cleanup_service.py @@ -9,6 +9,7 @@ import pytest from sqlalchemy import func, select from backend.app.models import Artist, ImageRecord, Tag, TagKind +from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage from backend.app.models.tag import image_tag from backend.app.services import cleanup_service @@ -376,7 +377,12 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_ {"image_record_id": img.id, "tag_id": c.id}, {"image_record_id": img.id, "tag_id": s.id}, # series membership ])) - db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1)) + ch = SeriesChapter(series_tag_id=s.id, chapter_number=1) + db_sync.add(ch) + db_sync.flush() + db_sync.add(SeriesPage( + series_tag_id=s.id, chapter_id=ch.id, image_id=img.id, page_number=1 + )) db_sync.commit() result = cleanup_service.reset_content_tagging(db_sync, dry_run=False) diff --git a/tests/test_series_chapters.py b/tests/test_series_chapters.py new file mode 100644 index 0000000..07aa767 --- /dev/null +++ b/tests/test_series_chapters.py @@ -0,0 +1,193 @@ +"""FC-6.1 — chapter layer over series_page.""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_chapter import SeriesChapter +from backend.app.models.series_page import SeriesPage +from backend.app.services.series_service import SeriesError, SeriesService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _img(db): + global _SEQ + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_sc_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +async def _series(db, name): + return (await TagService(db).find_or_create(name, TagKind.series)).id + + +async def _chapter_numbers(db, sid): + rows = ( + await db.execute( + select(SeriesChapter.id, SeriesChapter.chapter_number) + .where(SeriesChapter.series_tag_id == sid) + .order_by(SeriesChapter.chapter_number) + ) + ).all() + return [(r.id, r.chapter_number) for r in rows] + + +async def _page_order(db, chapter_id): + rows = ( + await db.execute( + select(SeriesPage.image_id) + .where(SeriesPage.chapter_id == chapter_id) + .order_by(SeriesPage.page_number) + ) + ).scalars().all() + return list(rows) + + +@pytest.mark.asyncio +async def test_add_images_creates_default_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C1") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2]) + chapters = await _chapter_numbers(db, sid) + assert len(chapters) == 1 + assert chapters[0][1] == 1 + assert await _page_order(db, chapters[0][0]) == [i1, i2] + + +@pytest.mark.asyncio +async def test_create_and_target_specific_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C2") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # default chapter 1 + ch2 = await svc.create_chapter(sid, title="Two") + assert ch2["chapter_number"] == 2 + await svc.add_images(sid, [i2], chapter_id=ch2["id"]) + assert await _page_order(db, ch2["id"]) == [i2] + # i1 stayed in chapter 1. + chapters = await _chapter_numbers(db, sid) + assert len(chapters) == 2 + + +@pytest.mark.asyncio +async def test_reorder_chapters(db): + svc = SeriesService(db) + sid = await _series(db, "C3") + c1 = (await svc.create_chapter(sid))["id"] + c2 = (await svc.create_chapter(sid))["id"] + c3 = (await svc.create_chapter(sid))["id"] + await svc.reorder_chapters(sid, [c3, c1, c2]) + assert await _chapter_numbers(db, sid) == [(c3, 1), (c1, 2), (c2, 3)] + with pytest.raises(SeriesError): + await svc.reorder_chapters(sid, [c1, c2]) + + +@pytest.mark.asyncio +async def test_reorder_pages_within_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C4") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2, i3]) + cid = (await _chapter_numbers(db, sid))[0][0] + await svc.reorder_pages(sid, cid, [i3, i1, i2]) + assert await _page_order(db, cid) == [i3, i1, i2] + + +@pytest.mark.asyncio +async def test_legacy_reorder_rejects_multi_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C5") + i1 = await _img(db) + await svc.add_images(sid, [i1]) + await svc.create_chapter(sid) # now 2 chapters + with pytest.raises(SeriesError): + await svc.reorder(sid, [i1]) + + +@pytest.mark.asyncio +async def test_merge_chapter_moves_pages_and_renumbers(db): + svc = SeriesService(db) + sid = await _series(db, "C6") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # chapter 1 + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2, i3], chapter_id=c2) + c1 = (await _chapter_numbers(db, sid))[0][0] + moved = await svc.merge_chapter(sid, c2, c1) + assert moved == 2 + assert await _page_order(db, c1) == [i1, i2, i3] + # source chapter gone; remaining renumbered 1..N. + assert await _chapter_numbers(db, sid) == [(c1, 1)] + + +@pytest.mark.asyncio +async def test_delete_chapter_cascades_pages_and_renumbers(db): + svc = SeriesService(db) + sid = await _series(db, "C7") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1]) + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2], chapter_id=c2) + await svc.delete_chapter(sid, c2) + assert [c for _, c in await _chapter_numbers(db, sid)] == [1] + # i2's page cascaded away with its chapter. + cnt = await db.scalar( + select(func.count()).select_from(SeriesPage) + .where(SeriesPage.image_id == i2) + ) + assert cnt == 0 + + +@pytest.mark.asyncio +async def test_set_cover_brings_chapter_and_page_to_front(db): + svc = SeriesService(db) + sid = await _series(db, "C8") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # chapter 1 + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2, i3], chapter_id=c2) + await svc.set_cover(sid, i3) + chapters = await _chapter_numbers(db, sid) + assert chapters[0][0] == c2 # i3's chapter is now first + assert (await _page_order(db, c2))[0] == i3 # i3 first in its chapter + + +@pytest.mark.asyncio +async def test_list_pages_flags_gaps_from_stated_ranges(db): + svc = SeriesService(db) + sid = await _series(db, "C9") + await svc.create_chapter(sid, stated_page_start=1, stated_page_end=4) + await svc.create_chapter(sid, stated_page_start=9, stated_page_end=12) + out = await svc.list_pages(sid) + assert len(out["chapters"]) == 2 + assert len(out["gaps"]) == 1 + assert out["gaps"][0]["start"] == 5 + assert out["gaps"][0]["end"] == 8 + + +@pytest.mark.asyncio +async def test_add_with_stated_pages_records_them(db): + svc = SeriesService(db) + sid = await _series(db, "C10") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2], stated_pages={i1: 9, i2: 10}) + rows = dict( + ( + await db.execute( + select(SeriesPage.image_id, SeriesPage.stated_page) + .where(SeriesPage.series_tag_id == sid) + ) + ).all() + ) + assert rows[i1] == 9 + assert rows[i2] == 10 diff --git a/tests/test_series_model.py b/tests/test_series_model.py index ff382c5..605bebf 100644 --- a/tests/test_series_model.py +++ b/tests/test_series_model.py @@ -3,6 +3,7 @@ from sqlalchemy import select from sqlalchemy.exc import IntegrityError from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage from backend.app.services.tag_service import TagService @@ -24,10 +25,20 @@ async def _img(db): @pytest.mark.asyncio +async def _chapter(db, series_tag_id): + ch = SeriesChapter(series_tag_id=series_tag_id, chapter_number=1) + db.add(ch) + await db.flush() + return ch.id + + async def test_series_page_roundtrip_and_unique_image(db): s = await TagService(db).find_or_create("Vol1", TagKind.series) i1 = await _img(db) - db.add(SeriesPage(series_tag_id=s.id, image_id=i1, page_number=1)) + ch = await _chapter(db, s.id) + db.add(SeriesPage( + series_tag_id=s.id, chapter_id=ch, image_id=i1, page_number=1 + )) await db.flush() got = await db.scalar( select(SeriesPage.page_number).where(SeriesPage.image_id == i1) @@ -35,6 +46,9 @@ async def test_series_page_roundtrip_and_unique_image(db): assert got == 1 # UNIQUE image_id: same image can't be a page twice (any series). s2 = await TagService(db).find_or_create("Vol2", TagKind.series) - db.add(SeriesPage(series_tag_id=s2.id, image_id=i1, page_number=1)) + ch2 = await _chapter(db, s2.id) + db.add(SeriesPage( + series_tag_id=s2.id, chapter_id=ch2, image_id=i1, page_number=1 + )) with pytest.raises(IntegrityError): await db.flush() diff --git a/tests/test_tag_merge.py b/tests/test_tag_merge.py index 50a839c..ed772c3 100644 --- a/tests/test_tag_merge.py +++ b/tests/test_tag_merge.py @@ -414,16 +414,27 @@ async def test_alias_create_does_not_clobber_existing(db): @pytest.mark.asyncio async def test_merge_repoints_series_pages(db): + from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage svc = TagService(db) a = await svc.find_or_create("SerA", TagKind.series) b = await svc.find_or_create("SerB", TagKind.series) img = await _img(db) - db.add(SeriesPage(series_tag_id=a.id, image_id=img, page_number=1)) + cha = SeriesChapter(series_tag_id=a.id, chapter_number=1) + db.add(cha) + await db.flush() + db.add(SeriesPage( + series_tag_id=a.id, chapter_id=cha.id, image_id=img, page_number=1 + )) await db.flush() await svc.merge(a.id, b.id) owner = await db.scalar( select(SeriesPage.series_tag_id).where(SeriesPage.image_id == img) ) assert owner == b.id + # The chapter moved to the survivor too, so the page's FK stayed valid. + ch_owner = await db.scalar( + select(SeriesChapter.series_tag_id).where(SeriesChapter.id == cha.id) + ) + assert ch_owner == b.id -- 2.52.0 From 8ad40da145998b961803fe0205b8678ecf891220 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 16:43:10 -0400 Subject: [PATCH 05/10] =?UTF-8?q?feat(series):=20chapter-aware=20manage=20?= =?UTF-8?q?view=20+=20reader=20=E2=80=94=20frontend=20(FC-6.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes FC-6.1: the series management UI now works in chapters. - SeriesManageView: chapters as cards (inline-rename, stated-page range inputs, move up/down, merge-into-previous, delete, pick-as-add-target), pages drag-reorder WITHIN a chapter, a "gap: N-M missing" badge between chapters with a stated-page hole, and Add chapter / Add placeholder. The picker adds the selection into the targeted chapter. - seriesManage store: chapter CRUD + reorderChapters/moveChapter/mergeChapter/ reorderPages actions; consumes chapters[]/gaps[]; addSelected targets a chapter. - Reader: page_number is now within-chapter, so anchors switched to a global `seq` (reading-order position) — fixes scroll/jump/active collisions across chapters — plus chapter-title dividers at each chapter boundary. - Updated seriesManage.spec to the chaptered store shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend/src/stores/seriesManage.js | 97 +++++++-- frontend/src/stores/seriesReader.js | 22 +- frontend/src/views/SeriesManageView.vue | 256 +++++++++++++++++++----- frontend/src/views/SeriesReaderView.vue | 41 ++-- frontend/test/seriesManage.spec.js | 62 ++++-- 5 files changed, 385 insertions(+), 93 deletions(-) diff --git a/frontend/src/stores/seriesManage.js b/frontend/src/stores/seriesManage.js index 3967b68..9f7e628 100644 --- a/frontend/src/stores/seriesManage.js +++ b/frontend/src/stores/seriesManage.js @@ -16,7 +16,10 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { const tagId = ref(null) const series = ref(null) - const pages = ref([]) // [{image_id, page_number, thumbnail_url}] + const chapters = ref([]) // [{id, chapter_number, title, is_placeholder, stated_page_start/end, pages:[...]}] + const gaps = ref([]) // [{after_chapter_id, start, end}] + const pageCount = ref(0) + const targetChapterId = ref(null) // which chapter the picker adds into const picker = ref([]) // gallery scroll results const pickerCursor = ref(null) const pickerSelection = ref([]) // image ids @@ -28,7 +31,14 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { try { const body = await api.get(`/api/series/${id}/pages`) series.value = body.series - pages.value = body.pages + chapters.value = body.chapters || [] + gaps.value = body.gaps || [] + pageCount.value = (body.pages || []).length + // Keep a valid add-target: the selected chapter, else the first one. + const ids = chapters.value.map(c => c.id) + if (!ids.includes(targetChapterId.value)) { + targetChapterId.value = ids.length ? ids[0] : null + } } finally { loading.value = false } @@ -38,6 +48,68 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { if (tagId.value != null) await load(tagId.value) } + function gapAfter(chapterId) { + return gaps.value.find(g => g.after_chapter_id === chapterId) || null + } + + // ---- chapters ---- + async function createChapter({ title = null, isPlaceholder = false } = {}) { + await api.post(`/api/series/${tagId.value}/chapters`, { + body: { title, is_placeholder: isPlaceholder } + }) + await refresh() + } + + async function renameChapter(chapterId, title) { + await api.patch(`/api/series/${tagId.value}/chapters/${chapterId}`, { + body: { title } + }) + await refresh() + } + + async function setChapterStated(chapterId, start, end) { + await api.patch(`/api/series/${tagId.value}/chapters/${chapterId}`, { + body: { stated_page_start: start, stated_page_end: end } + }) + await refresh() + } + + async function reorderChapters(orderedChapterIds) { + await api.post(`/api/series/${tagId.value}/chapters/reorder`, { + body: { chapter_ids: orderedChapterIds } + }) + await refresh() + } + + async function moveChapter(chapterId, dir) { + const ids = chapters.value.map(c => c.id) + const from = ids.indexOf(chapterId) + const to = from + dir + if (from === -1 || to < 0 || to >= ids.length) return + await reorderChapters(moveItem(ids, from, to)) + } + + async function deleteChapter(chapterId) { + await api.delete(`/api/series/${tagId.value}/chapters/${chapterId}`) + await refresh() + } + + async function mergeChapter(sourceId, targetId) { + await api.post(`/api/series/${tagId.value}/chapters/${sourceId}/merge`, { + body: { target_chapter_id: targetId } + }) + await refresh() + toast({ text: 'Chapters merged', type: 'success' }) + } + + async function reorderPages(chapterId, orderedImageIds) { + await api.post(`/api/series/${tagId.value}/chapters/${chapterId}/reorder`, { + body: { image_ids: orderedImageIds } + }) + await refresh() + } + + // ---- picker / pages ---- async function loadPicker(reset = false) { if (reset) { picker.value = []; pickerCursor.value = null } const params = { limit: 50 } @@ -55,12 +127,13 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { async function addSelected() { if (pickerSelection.value.length === 0) return + if (targetChapterId.value == null) await createChapter() await api.post(`/api/series/${tagId.value}/pages`, { - body: { image_ids: pickerSelection.value } + body: { image_ids: pickerSelection.value, chapter_id: targetChapterId.value } }) pickerSelection.value = [] await refresh() - toast({ text: 'Added to series', type: 'success' }) + toast({ text: 'Added to chapter', type: 'success' }) } async function remove(imageId) { @@ -70,13 +143,6 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { await refresh() } - async function reorder(orderedImageIds) { - await api.post(`/api/series/${tagId.value}/reorder`, { - body: { image_ids: orderedImageIds } - }) - await refresh() - } - async function setCover(imageId) { await api.post(`/api/series/${tagId.value}/cover`, { body: { image_id: imageId } @@ -85,8 +151,11 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { } return { - tagId, series, pages, picker, pickerCursor, pickerSelection, loading, - load, refresh, loadPicker, togglePick, addSelected, remove, - reorder, setCover + tagId, series, chapters, gaps, pageCount, targetChapterId, + picker, pickerCursor, pickerSelection, loading, + load, refresh, gapAfter, + createChapter, renameChapter, setChapterStated, reorderChapters, + moveChapter, deleteChapter, mergeChapter, reorderPages, + loadPicker, togglePick, addSelected, remove, setCover } }) diff --git a/frontend/src/stores/seriesReader.js b/frontend/src/stores/seriesReader.js index beac66d..86f2e98 100644 --- a/frontend/src/stores/seriesReader.js +++ b/frontend/src/stores/seriesReader.js @@ -42,7 +42,27 @@ export const useSeriesReaderStore = defineStore('seriesReader', () => { await run(async () => { const body = await api.get(`/api/series/${tagId}/pages`) series.value = body.series - pages.value = body.pages + // page_number is now WITHIN a chapter, so it can't anchor scroll/jump + // (two chapters both have a page 1). Decorate each page with a global + // `seq` (reading-order position) for anchors, plus chapter-divider info + // so the reader can mark where each chapter begins. + const chapters = body.chapters || [] + const labelById = {} + for (const c of chapters) { + labelById[c.id] = c.title || `Chapter ${c.chapter_number}` + } + let prevChapter = null + pages.value = (body.pages || []).map((p, i) => { + const isChapterStart = + chapters.length > 1 && p.chapter_id !== prevChapter + prevChapter = p.chapter_id + return { + ...p, + seq: i + 1, + isChapterStart, + chapterLabel: labelById[p.chapter_id] || null + } + }) }) } diff --git a/frontend/src/views/SeriesManageView.vue b/frontend/src/views/SeriesManageView.vue index d12ddcc..52feba1 100644 --- a/frontend/src/views/SeriesManageView.vue +++ b/frontend/src/views/SeriesManageView.vue @@ -2,9 +2,11 @@
{{ store.series?.name || 'Series' }} - {{ store.pages.length }} page(s) + + {{ store.chapters.length }} chapter(s) · {{ store.pageCount }} page(s) +
-
-
- {{ p.page_number }} - -
- - + +
+ + +
+ No chapters yet — add one, then add images from the right.
-
- No pages yet — add images from the right. + +
+ Add chapter + + Add placeholder +
+
{{ store.pickerSelection.length }} selected @@ -42,7 +139,7 @@ size="small" color="accent" variant="flat" :disabled="store.pickerSelection.length === 0" @click="store.addSelected()" - >Add to series + >Add to {{ targetLabel }}
diff --git a/frontend/src/router.js b/frontend/src/router.js index 9a6a05d..2073c84 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -4,6 +4,7 @@ import GalleryView from './views/GalleryView.vue' import ShowcaseView from './views/ShowcaseView.vue' import TagsView from './views/TagsView.vue' import ArtistView from './views/ArtistView.vue' +import SeriesView from './views/SeriesView.vue' import SeriesManageView from './views/SeriesManageView.vue' import SeriesReaderView from './views/SeriesReaderView.vue' import SubscriptionsView from './views/SubscriptionsView.vue' @@ -25,7 +26,9 @@ const routes = [ { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, // Artist detail — no meta.title (reached by clicking an artist, not nav). { path: '/artist/:slug', name: 'artist', component: ArtistView }, - // Series management — no meta.title (reached from a series tag card). + // Series browse — a nav entry (meta.title). Peer of Posts. + { path: '/series', name: 'series', component: SeriesView, meta: { title: 'Series' } }, + // Series management — no meta.title (reached from a series card/tag). { path: '/series/:tagId', name: 'series-manage', component: SeriesManageView }, // Series reader — immersive (no top nav, no meta.title). { path: '/series/:tagId/read', name: 'series-read', component: SeriesReaderView, meta: { immersive: true } }, diff --git a/frontend/src/stores/seriesBrowse.js b/frontend/src/stores/seriesBrowse.js new file mode 100644 index 0000000..adb796c --- /dev/null +++ b/frontend/src/stores/seriesBrowse.js @@ -0,0 +1,31 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' +import { useAsyncAction } from '../composables/useAsyncAction.js' + +// Backs the Series browse view (FC-6.2). list_series returns every series as a +// card; homelab scale is small enough to load the whole list and sort/filter +// server-side. +export const useSeriesBrowseStore = defineStore('seriesBrowse', () => { + const api = useApi() + const series = ref([]) + const sort = ref('recent') + const artistId = ref(null) + const { loading, error, run } = useAsyncAction({ errorAs: 'message' }) + + async function load() { + await run(async () => { + const params = { sort: sort.value } + if (artistId.value != null) params.artist_id = artistId.value + const body = await api.get('/api/series', { params }) + series.value = body.series || [] + }) + } + + function setSort(s) { + sort.value = s + return load() + } + + return { series, sort, artistId, loading, error, load, setSort } +}) diff --git a/frontend/src/views/SeriesView.vue b/frontend/src/views/SeriesView.vue new file mode 100644 index 0000000..c135848 --- /dev/null +++ b/frontend/src/views/SeriesView.vue @@ -0,0 +1,145 @@ + + + + + -- 2.52.0 From c0fd80e694b271f34582ce5dfc6b2de0fe304c11 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 18:58:18 -0400 Subject: [PATCH 08/10] =?UTF-8?q?feat(series):=20assisted-continuation=20m?= =?UTF-8?q?atcher=20+=20suggestion=20queue=20=E2=80=94=20backend=20(FC-6.3?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirm-only "this post may continue this series" matcher. - series_suggestion table (post_id, series_tag_id, score, signals jsonb, status pending|added|dismissed, UNIQUE(post,series)); migration 0041 + two settings knobs (series_suggest_enabled, series_suggest_threshold). - series_match_service: weighted additive score (title-stem / same-artist / page-continuity / shared-distinctive-tags), no single signal gating. The title "pattern" is derived on the fly from the post titles already in a series, so it sharpens as more are confirmed (no persisted state to drift). Candidates are bounded to the post's artist. match_post upserts pending suggestions (UNIQUE + on-conflict, respecting prior added/dismissed decisions). - accept reuses add_post_as_chapter then marks 'added'; dismiss marks 'dismissed'. - rescan_series_suggestions_task: settings-gated, time-boxed + self-resuming from a post-id cursor (maintenance_long lane), like normalize_tags_task. - API: GET /series/suggestions, POST ...//accept|dismiss, POST .../rescan. - Settings: enabled + threshold exposed via /settings/import. - Tests: pure scoring helpers + matcher/accept/dismiss/rescan lifecycle + UNIQUE dedup. Frontend (Suggestions tab + settings card) lands next. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0041_series_suggestions.py | 98 ++++++ backend/app/api/settings.py | 17 + backend/app/api/tags.py | 41 +++ backend/app/models/__init__.py | 2 + backend/app/models/import_settings.py | 9 + backend/app/models/series_suggestion.py | 55 +++ backend/app/services/series_match_service.py | 337 +++++++++++++++++++ backend/app/tasks/admin.py | 51 +++ tests/test_series_match.py | 48 +++ tests/test_series_match_service.py | 127 +++++++ 10 files changed, 785 insertions(+) create mode 100644 alembic/versions/0041_series_suggestions.py create mode 100644 backend/app/models/series_suggestion.py create mode 100644 backend/app/services/series_match_service.py create mode 100644 tests/test_series_match.py create mode 100644 tests/test_series_match_service.py diff --git a/alembic/versions/0041_series_suggestions.py b/alembic/versions/0041_series_suggestions.py new file mode 100644 index 0000000..51b690a --- /dev/null +++ b/alembic/versions/0041_series_suggestions.py @@ -0,0 +1,98 @@ +"""series suggestions: assisted-continuation matcher (FC-6.3) + +Revision ID: 0041 +Revises: 0040 +Create Date: 2026-06-07 + +A confirm-only queue of "this post may continue this series" hints, plus two +import_settings knobs (enable + score threshold) for the matcher. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0041" +down_revision: Union[str, None] = "0040" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_suggestion", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "post_id", + sa.Integer, + sa.ForeignKey("post.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("score", sa.Float, nullable=False), + sa.Column("signals", sa.JSON, nullable=True), + sa.Column( + "status", sa.String(16), nullable=False, server_default="pending" + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + op.create_index( + "ix_series_suggestion_post_id", "series_suggestion", ["post_id"] + ) + op.create_index( + "ix_series_suggestion_series_tag_id", + "series_suggestion", + ["series_tag_id"], + ) + op.create_index( + "ix_series_suggestion_status", "series_suggestion", ["status"] + ) + + op.add_column( + "import_settings", + sa.Column( + "series_suggest_enabled", + sa.Boolean, + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "series_suggest_threshold", + sa.Float, + nullable=False, + server_default="0.5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "series_suggest_threshold") + op.drop_column("import_settings", "series_suggest_enabled") + op.drop_index("ix_series_suggestion_status", table_name="series_suggestion") + op.drop_index( + "ix_series_suggestion_series_tag_id", table_name="series_suggestion" + ) + op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion") + op.drop_table("series_suggestion") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index fa97bda..7d69fbf 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -25,6 +25,8 @@ _EDITABLE_FIELDS = ( "download_schedule_default_seconds", "download_event_retention_days", "download_failure_warning_threshold", + "series_suggest_enabled", + "series_suggest_threshold", ) @@ -46,6 +48,8 @@ async def get_import_settings(): "download_schedule_default_seconds": row.download_schedule_default_seconds, "download_event_retention_days": row.download_event_retention_days, "download_failure_warning_threshold": row.download_failure_warning_threshold, + "series_suggest_enabled": row.series_suggest_enabled, + "series_suggest_threshold": row.series_suggest_threshold, }) @@ -96,6 +100,19 @@ async def update_import_settings(): if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100: return _bad_int("download_failure_warning_threshold", 1, 100) + if "series_suggest_enabled" in body and not isinstance( + body["series_suggest_enabled"], bool + ): + return jsonify( + {"error": "series_suggest_enabled must be a boolean"} + ), 400 + if "series_suggest_threshold" in body: + v = body["series_suggest_threshold"] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: + return jsonify( + {"error": "series_suggest_threshold must be a number in [0, 1]"} + ), 400 + async with get_session() as session: row = await ImportSettings.load(session) for field in _EDITABLE_FIELDS: diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index cfbe4b7..918fa42 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -8,6 +8,7 @@ from ..extensions import get_session from ..models import Tag, TagKind from ..models.tag_allowlist import TagAllowlist from ..services.bulk_tag_service import BulkTagService +from ..services.series_match_service import SeriesMatchService from ..services.series_service import SeriesError, SeriesService from ..services.tag_directory_service import TagDirectoryService from ..services.tag_service import ( @@ -653,3 +654,43 @@ async def series_add_post(tag_id: int): return _series_err(exc) await session.commit() return jsonify(out) + + +# ---- suggestion queue (FC-6.3) -------------------------------------------- + + +@tags_bp.route("/series/suggestions", methods=["GET"]) +async def series_suggestions_list(): + async with get_session() as session: + rows = await SeriesMatchService(session).list_pending() + return jsonify({"suggestions": rows}) + + +@tags_bp.route("/series/suggestions//accept", methods=["POST"]) +async def series_suggestion_accept(sid: int): + async with get_session() as session: + try: + out = await SeriesMatchService(session).accept(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(out) + + +@tags_bp.route("/series/suggestions//dismiss", methods=["POST"]) +async def series_suggestion_dismiss(sid: int): + async with get_session() as session: + try: + await SeriesMatchService(session).dismiss(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route("/series/suggestions/rescan", methods=["POST"]) +async def series_suggestions_rescan(): + from ..tasks.admin import rescan_series_suggestions_task + + res = rescan_series_suggestions_task.delay() + return jsonify({"task_id": res.id}) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index bfde14b..00047f9 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -20,6 +20,7 @@ from .post import Post from .post_attachment import PostAttachment from .series_chapter import SeriesChapter from .series_page import SeriesPage +from .series_suggestion import SeriesSuggestion from .source import Source from .tag import Tag, TagKind, image_tag from .tag_alias import TagAlias @@ -42,6 +43,7 @@ __all__ = [ "PostAttachment", "SeriesChapter", "SeriesPage", + "SeriesSuggestion", "ImageRecord", "ImageProvenance", "Tag", diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index e5142ca..8c59fa8 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -64,6 +64,15 @@ class ImportSettings(Base): Integer, nullable=False, default=3, ) + # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is + # the weighted-score cut-off (0..1) above which a pending suggestion is made. + series_suggest_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, + ) + series_suggest_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.5, + ) + @classmethod async def load(cls, session) -> ImportSettings: """The singleton settings row (id=1), via an async session.""" diff --git a/backend/app/models/series_suggestion.py b/backend/app/models/series_suggestion.py new file mode 100644 index 0000000..316483b --- /dev/null +++ b/backend/app/models/series_suggestion.py @@ -0,0 +1,55 @@ +"""SeriesSuggestion — a confirm-only "this post may continue this series" hint. + +The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted +signals and, above the configured threshold, records a pending suggestion. The +operator confirms (→ the post is added as a chapter) or dismisses it; FC never +files a post into a series on its own. status is a plain string (no Postgres +ENUM — see the check-existing-enums lesson): pending | added | dismissed. +""" + +from datetime import datetime + +from sqlalchemy import ( + JSON, + DateTime, + Float, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesSuggestion(Base): + __tablename__ = "series_suggestion" + __table_args__ = ( + UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + score: Mapped[float] = mapped_column(Float, nullable=False) + signals: Mapped[dict | None] = mapped_column(JSON, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending", index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/services/series_match_service.py b/backend/app/services/series_match_service.py new file mode 100644 index 0000000..364229f --- /dev/null +++ b/backend/app/services/series_match_service.py @@ -0,0 +1,337 @@ +"""FC-6.3 — assisted continuation matcher. + +Scores a (post, candidate series) pair from several weighted signals; above the +configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the +operator accepts (post becomes a chapter) or dismisses. No single signal gates; +the score is an additive weighted sum, each signal a 0..1 strength. + +The learned "title pattern" isn't persisted — it's derived on the fly from the +post titles already in a series, so it sharpens automatically as more posts are +confirmed into the series. +""" + +import re +import time + +from sqlalchemy import and_, func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord, Post, Tag, TagKind +from ..models.series_chapter import SeriesChapter +from ..models.series_page import SeriesPage +from ..models.series_suggestion import SeriesSuggestion +from ..models.tag import image_tag +from .page_number_parser import parse_page_range +from .series_service import SeriesError, SeriesService + +# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants; +# only the on/off + threshold are operator-tunable (sensitivity is the knob +# that matters; per-signal weights are an over-tune for v1). +WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15} + +_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom) +_MAX_CANDIDATES = 50 + +# Strip installment markers so titles collapse to their stable stem. +_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I) +_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]") +_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$") + + +def normalize_title(title: str | None) -> str: + if not title: + return "" + t = _PAGE_TOKEN.sub("", title) + t = _BRACKET_NUM.sub("", t) + t = _TRAILING_NUM.sub("", t) + return " ".join(t.split()).strip().lower() + + +def _common_prefix(strings: list[str]) -> str: + strings = [s for s in strings if s] + if not strings: + return "" + pre = strings[0] + for s in strings[1:]: + i = 0 + while i < len(pre) and i < len(s) and pre[i] == s[i]: + i += 1 + pre = pre[:i] + if not pre: + break + return pre.strip() + + +def title_signal(series_titles: list[str], post_title: str | None) -> float: + """Overlap of the post title against the series' title stem (the longest + common prefix of its known titles, page/installment markers removed).""" + norm = [normalize_title(t) for t in series_titles] + norm = [t for t in norm if t] + pt = normalize_title(post_title) + if not norm or not pt: + return 0.0 + stem = _common_prefix(norm) + if len(stem) < 3: + stem = max(norm, key=len) + i = 0 + while i < len(stem) and i < len(pt) and stem[i] == pt[i]: + i += 1 + return min(1.0, i / max(len(stem), 4)) + + +def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float: + """1.0 when the post's first page continues right after the series' last + stated page; partial for a near-continuation; 0 otherwise.""" + if series_max_stated_end is None or post_start is None: + return 0.0 + diff = post_start - series_max_stated_end + if diff == 1: + return 1.0 + if 2 <= diff <= 3: + return 0.5 + return 0.0 + + +def tags_signal(shared_distinctive: int) -> float: + if shared_distinctive <= 0: + return 0.0 + return min(1.0, shared_distinctive / 3.0) + + +def weighted_score(signals: dict) -> float: + return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) + + +class SeriesMatchService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _post_image_ids(self, post_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id) + ) + ).scalars().all() + return list(rows) + + async def _series_image_ids(self, series_tag_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesPage.image_id).where( + SeriesPage.series_tag_id == series_tag_id + ) + ) + ).scalars().all() + return set(rows) + + async def _series_post_titles(self, series_tag_id: int) -> list[str]: + rows = ( + await self.session.execute( + select(Post.post_title) + .select_from(SeriesPage) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .join(Post, Post.id == ImageRecord.primary_post_id) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + Post.post_title.isnot(None), + ) + ) + .distinct() + ) + ).scalars().all() + return [t for t in rows if t] + + async def _series_max_stated_end(self, series_tag_id: int) -> int | None: + return await self.session.scalar( + select(func.max(SeriesChapter.stated_page_end)).where( + SeriesChapter.series_tag_id == series_tag_id + ) + ) + + async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]: + ids = list(image_ids) + if not ids: + return set() + rows = ( + await self.session.execute( + select(image_tag.c.tag_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where( + and_( + image_tag.c.image_record_id.in_(ids), + Tag.kind.in_(_DISTINCTIVE_KINDS), + ) + ) + .distinct() + ) + ).scalars().all() + return set(rows) + + async def _candidate_series(self, artist_id: int) -> list[int]: + """Series that already contain a page by this artist — cross-artist + series are rare, so same-artist is the candidate bound.""" + rows = ( + await self.session.execute( + select(SeriesPage.series_tag_id) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(ImageRecord.artist_id == artist_id) + .distinct() + ) + ).scalars().all() + return list(rows) + + async def _decided_series(self, post_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesSuggestion.series_tag_id).where( + and_( + SeriesSuggestion.post_id == post_id, + SeriesSuggestion.status.in_(["added", "dismissed"]), + ) + ) + ) + ).scalars().all() + return set(rows) + + async def match_post(self, post_id: int, *, threshold: float) -> int: + """Score a post against its artist's series; upsert pending suggestions + for those at/above threshold. Returns the number written.""" + post = await self.session.get(Post, post_id) + if post is None or post.artist_id is None: + return 0 + post_images = await self._post_image_ids(post_id) + if not post_images: + return 0 + post_image_set = set(post_images) + rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}") + post_start = rng[0] if rng else None + post_dtags = await self._distinctive_tags(post_images) + + decided = await self._decided_series(post_id) + candidates = await self._candidate_series(post.artist_id) + made = 0 + for sid in candidates[:_MAX_CANDIDATES]: + if sid in decided: + continue + s_images = await self._series_image_ids(sid) + if post_image_set & s_images: + continue # the post is already (partly) in this series + signals = { + "title": title_signal( + await self._series_post_titles(sid), post.post_title + ), + "artist": 1.0, # candidates are same-artist by construction + "pages": pages_signal( + await self._series_max_stated_end(sid), post_start + ), + "tags": tags_signal( + len(post_dtags & await self._distinctive_tags(s_images)) + ), + } + score = weighted_score(signals) + if score < threshold: + continue + await self.session.execute( + pg_insert(SeriesSuggestion) + .values( + post_id=post_id, series_tag_id=sid, + score=score, signals=signals, status="pending", + ) + .on_conflict_do_update( + constraint="uq_series_suggestion_post_series", + set_={"score": score, "signals": signals, "status": "pending"}, + where=SeriesSuggestion.status == "pending", + ) + ) + made += 1 + return made + + # ---- queue ops -------------------------------------------------------- + + async def list_pending(self) -> list[dict]: + rows = ( + await self.session.execute( + select( + SeriesSuggestion.id, + SeriesSuggestion.post_id, + SeriesSuggestion.series_tag_id, + SeriesSuggestion.score, + SeriesSuggestion.signals, + Post.post_title, + Post.external_post_id, + Tag.name.label("series_name"), + ) + .join(Post, Post.id == SeriesSuggestion.post_id) + .join(Tag, Tag.id == SeriesSuggestion.series_tag_id) + .where(SeriesSuggestion.status == "pending") + .order_by(SeriesSuggestion.score.desc()) + ) + ).all() + return [ + { + "id": r.id, + "post_id": r.post_id, + "series_tag_id": r.series_tag_id, + "series_name": r.series_name, + "post_title": r.post_title or f"Post {r.external_post_id}", + "score": r.score, + "signals": r.signals or {}, + } + for r in rows + ] + + async def accept(self, suggestion_id: int) -> dict: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status != "pending": + raise SeriesError(f"suggestion {suggestion_id} is already {s.status}") + out = await SeriesService(self.session).add_post_as_chapter( + s.series_tag_id, s.post_id + ) + s.status = "added" + self.session.add(s) + return out + + async def dismiss(self, suggestion_id: int) -> None: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status == "pending": + s.status = "dismissed" + self.session.add(s) + + async def rescan( + self, *, threshold: float, time_budget_seconds: float | None = None, + after_post_id: int = 0, + ) -> dict: + """Score every post (id > after_post_id) against its artist's series, + time-boxed + resumable like the other long maintenance sweeps.""" + post_ids = ( + await self.session.execute( + select(Post.id) + .where(Post.id > after_post_id) + .order_by(Post.id.asc()) + ) + ).scalars().all() + summary = { + "scanned": 0, "suggested": 0, + "partial": False, "resume_after_id": after_post_id, + } + start = time.monotonic() + for pid in post_ids: + summary["scanned"] += 1 + summary["resume_after_id"] = pid + summary["suggested"] += await self.match_post(pid, threshold=threshold) + await self.session.commit() # commit per post so progress survives + if ( + time_budget_seconds is not None + and time.monotonic() - start >= time_budget_seconds + ): + summary["partial"] = True + break + else: + summary["partial"] = False + return summary diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 4942c26..8096707 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -147,3 +147,54 @@ def normalize_tags_task(self) -> dict: ) normalize_tags_task.delay() return summary + + +# Time-box one rescan chunk well under the soft limit and re-enqueue from the +# cursor — scoring every post against its artist's series is O(posts) and grows +# with the library (FC-6.3). Mirrors normalize_tags_task. +_SERIES_RESCAN_CHUNK_SECONDS = 600 + + +@celery.task( + name="backend.app.tasks.admin.rescan_series_suggestions_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict: + """Score posts against their artist's series and write pending suggestions + (FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor. + Per-task async engine (NullPool) under its own asyncio loop, like normalize.""" + import asyncio + + from ..models import ImportSettings + from ..services.series_match_service import SeriesMatchService + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + try: + async with async_factory() as session: + settings = await ImportSettings.load(session) + if not settings.series_suggest_enabled: + return {"skipped": "series suggestions disabled"} + threshold = settings.series_suggest_threshold + return await SeriesMatchService(session).rescan( + threshold=threshold, + time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS, + after_post_id=after_post_id, + ) + finally: + await async_engine.dispose() + + summary = asyncio.run(_run()) + if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id: + log.info( + "rescan_series_suggestions chunk done (%d scanned, %d suggested, " + "resume after %s) — re-enqueuing", + summary.get("scanned", 0), summary.get("suggested", 0), + summary["resume_after_id"], + ) + rescan_series_suggestions_task.delay(summary["resume_after_id"]) + return summary diff --git a/tests/test_series_match.py b/tests/test_series_match.py new file mode 100644 index 0000000..38be348 --- /dev/null +++ b/tests/test_series_match.py @@ -0,0 +1,48 @@ +"""Pure scoring helpers for the series continuation matcher (FC-6.3).""" + +from backend.app.services.series_match_service import ( + normalize_title, + pages_signal, + tags_signal, + title_signal, + weighted_score, +) + + +def test_normalize_title_strips_installment_markers(): + assert normalize_title("Story pages 9-12") == "story" + assert normalize_title("Bunker Diaries 04") == "bunker diaries" + assert normalize_title("My Comic [3/8]") == "my comic" + assert normalize_title("") == "" + assert normalize_title(None) == "" + + +def test_title_signal(): + # Same stem after stripping page markers → full match. + assert title_signal(["Story pages 1-4"], "Story pages 9-12") == 1.0 + # Unrelated → no overlap. + assert title_signal(["My Comic"], "Other Thing") == 0.0 + # No data → 0. + assert title_signal([], "x") == 0.0 + + +def test_pages_signal(): + assert pages_signal(8, 9) == 1.0 # clean continuation + assert pages_signal(8, 10) == 0.5 # near (diff 2) + assert pages_signal(8, 11) == 0.5 # near (diff 3) + assert pages_signal(8, 20) == 0.0 # far + assert pages_signal(8, 5) == 0.0 # before / overlap + assert pages_signal(None, 9) == 0.0 + + +def test_tags_signal(): + assert tags_signal(0) == 0.0 + assert tags_signal(3) == 1.0 + assert tags_signal(6) == 1.0 # capped + assert round(tags_signal(1), 3) == 0.333 + + +def test_weighted_score(): + assert weighted_score({"title": 1, "artist": 1, "pages": 1, "tags": 1}) == 1.0 + assert weighted_score({}) == 0.0 + assert weighted_score({"title": 1.0, "artist": 1.0}) == 0.6 diff --git a/tests/test_series_match_service.py b/tests/test_series_match_service.py new file mode 100644 index 0000000..5e4970e --- /dev/null +++ b/tests/test_series_match_service.py @@ -0,0 +1,127 @@ +"""FC-6.3 — matcher writes pending suggestions; accept/dismiss lifecycle.""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import Artist, ImageRecord, Post, TagKind +from backend.app.models.series_suggestion import SeriesSuggestion +from backend.app.services.series_match_service import SeriesMatchService +from backend.app.services.series_service import SeriesService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _artist(db, name): + a = Artist(name=name, slug=name.lower().replace(" ", "-")) + db.add(a) + await db.flush() + return a + + +async def _post_with_images(db, artist, title, ext, n): + global _SEQ + p = Post(artist_id=artist.id, external_post_id=ext, post_title=title) + db.add(p) + await db.flush() + for _ in range(n): + _SEQ += 1 + db.add(ImageRecord( + path=f"/tmp/fc_sm_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="downloaded", + primary_post_id=p.id, artist_id=artist.id, + )) + await db.flush() + return p + + +async def _series_from(db, post): + out = await SeriesService(db).promote_post_to_series(post.id) + return out["series_tag_id"] + + +@pytest.mark.asyncio +async def test_match_post_suggests_continuation(db): + artist = await _artist(db, "Story Artist") + a = await _post_with_images(db, artist, "Story pages 1-4", "a", 4) + sid = await _series_from(db, a) # series stated 1-4 + b = await _post_with_images(db, artist, "Story pages 5-8", "b", 4) + + svc = SeriesMatchService(db) + made = await svc.match_post(b.id, threshold=0.5) + assert made == 1 + + pending = await svc.list_pending() + assert len(pending) == 1 + sug = pending[0] + assert sug["series_tag_id"] == sid + assert sug["post_id"] == b.id + # title 1.0 + artist 1.0 + pages 1.0 (5 == 4+1) → 0.4+0.2+0.25 = 0.85 + assert sug["score"] >= 0.85 + assert sug["signals"]["pages"] == 1.0 + + +@pytest.mark.asyncio +async def test_match_is_idempotent_under_unique(db): + artist = await _artist(db, "Idem Artist") + a = await _post_with_images(db, artist, "Tale pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Tale pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + await svc.match_post(b.id, threshold=0.5) # again + cnt = await db.scalar( + select(func.count()).select_from(SeriesSuggestion) + .where(SeriesSuggestion.post_id == b.id) + ) + assert cnt == 1 + + +@pytest.mark.asyncio +async def test_accept_adds_chapter_and_marks_added(db): + artist = await _artist(db, "Accept Artist") + a = await _post_with_images(db, artist, "Saga pages 1-4", "a", 3) + sid = await _series_from(db, a) + b = await _post_with_images(db, artist, "Saga pages 5-8", "b", 3) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.accept(sug["id"]) + data = await SeriesService(db).list_pages(sid) + assert len(data["chapters"]) == 2 # b became a 2nd chapter + status = await db.scalar( + select(SeriesSuggestion.status).where(SeriesSuggestion.id == sug["id"]) + ) + assert status == "added" + + +@pytest.mark.asyncio +async def test_dismiss_then_rematch_skips(db): + artist = await _artist(db, "Dismiss Artist") + a = await _post_with_images(db, artist, "Epic pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Epic pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.dismiss(sug["id"]) + assert await svc.list_pending() == [] + # A re-match must respect the dismissal (decided → skipped). + made = await svc.match_post(b.id, threshold=0.5) + assert made == 0 + + +@pytest.mark.asyncio +async def test_rescan_scans_posts(db): + artist = await _artist(db, "Rescan Artist") + a = await _post_with_images(db, artist, "Quest pages 1-4", "a", 2) + await _series_from(db, a) + await _post_with_images(db, artist, "Quest pages 5-8", "b", 2) + svc = SeriesMatchService(db) + summary = await svc.rescan(threshold=0.5) + assert summary["scanned"] >= 2 + assert summary["suggested"] >= 1 + assert summary["partial"] is False -- 2.52.0 From 19a91a1641c6c4e5296f4413397b22fb6815af04 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 19:02:19 -0400 Subject: [PATCH 09/10] =?UTF-8?q?feat(series):=20Suggestions=20tab=20+=20m?= =?UTF-8?q?atcher=20controls=20=E2=80=94=20frontend=20(FC-6.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes FC-6.3 with the UI. - SeriesView gains tabs: Browse (the existing grid) + Suggestions. - Suggestions tab: pending matches as rows (post → series, per-signal strength chips, score), Add (→ chapter) / Skip (→ dismiss); a "Matching on" toggle and a threshold field (both DB-backed via /settings/import), and a Rescan button that enqueues the background matcher. - seriesSuggestions store wires load / accept / dismiss / rescan / settings. Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend/src/stores/seriesSuggestions.js | 75 +++++++ frontend/src/views/SeriesView.vue | 239 +++++++++++++++++------ 2 files changed, 254 insertions(+), 60 deletions(-) create mode 100644 frontend/src/stores/seriesSuggestions.js diff --git a/frontend/src/stores/seriesSuggestions.js b/frontend/src/stores/seriesSuggestions.js new file mode 100644 index 0000000..281c666 --- /dev/null +++ b/frontend/src/stores/seriesSuggestions.js @@ -0,0 +1,75 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' +import { useAsyncAction } from '../composables/useAsyncAction.js' +import { toast } from '../utils/toast.js' + +// Backs the Series → Suggestions tab (FC-6.3). The matcher is confirm-only: +// accept turns a suggestion into a chapter; skip dismisses it. enabled + +// threshold are the operator-tunable knobs (DB-backed via /settings/import). +export const useSeriesSuggestionsStore = defineStore('seriesSuggestions', () => { + const api = useApi() + const suggestions = ref([]) + const enabled = ref(true) + const threshold = ref(0.5) + const { loading, error, run } = useAsyncAction({ errorAs: 'message' }) + + async function load() { + await run(async () => { + const body = await api.get('/api/series/suggestions') + suggestions.value = body.suggestions || [] + }) + } + + async function loadSettings() { + const s = await api.get('/api/settings/import') + enabled.value = s.series_suggest_enabled + threshold.value = s.series_suggest_threshold + } + + async function saveSettings(patch) { + await api.patch('/api/settings/import', { body: patch }) + } + + async function setEnabled(v) { + enabled.value = v + await saveSettings({ series_suggest_enabled: v }) + } + + async function setThreshold(v) { + threshold.value = v + await saveSettings({ series_suggest_threshold: v }) + } + + async function accept(id) { + try { + await api.post(`/api/series/suggestions/${id}/accept`, {}) + suggestions.value = suggestions.value.filter(s => s.id !== id) + toast({ text: 'Added to series', type: 'success' }) + } catch (e) { + toast({ text: `Accept failed: ${e.message}`, type: 'error' }) + } + } + + async function dismiss(id) { + try { + await api.post(`/api/series/suggestions/${id}/dismiss`, {}) + suggestions.value = suggestions.value.filter(s => s.id !== id) + } catch (e) { + toast({ text: `Skip failed: ${e.message}`, type: 'error' }) + } + } + + async function rescan() { + await api.post('/api/series/suggestions/rescan', {}) + toast({ + text: 'Rescan queued — runs in the background; reload to see new matches.', + type: 'info' + }) + } + + return { + suggestions, enabled, threshold, loading, error, + load, loadSettings, setEnabled, setThreshold, accept, dismiss, rescan + } +}) diff --git a/frontend/src/views/SeriesView.vue b/frontend/src/views/SeriesView.vue index c135848..cb3e754 100644 --- a/frontend/src/views/SeriesView.vue +++ b/frontend/src/views/SeriesView.vue @@ -1,77 +1,154 @@ -- 2.52.0 From daaa7543a8ed273ec120d856cf2e4bc72c95b6b4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 19:20:16 -0400 Subject: [PATCH 10/10] fix(backup,tags): unwedge backups on NFS (#739) + tag-standardize "0 groups" (#740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #739 — DB backups hung on NFS in uninterruptible D-state, defeating the 12-min subprocess timeout AND Celery's hard limit, so a stuck pg_dump held the concurrency-1 maintenance_long lane for hours — starving normalize_tags, re-extract, audits, and the new series rescan (which is why #740 "never applied"). Three fixes: - _run_bounded: Popen + bounded post-kill reap; if the child is unkillable (D-state) we stop waiting and re-raise TimeoutExpired, freeing the slot. The orphan is reaped by the OS once its syscall clears. - backup_db dumps to a LOCAL temp file then moves the finished .sql to the (NFS) _backups dir — pg_dump's long phase is now a DB-socket wait + local writes (killable) instead of an NFS write that hangs. backup_images keeps bounded-kill (too big to stage locally). - recover_stalled_backup_runs: split the stall window — db 40 min (was sharing images' 7h), so a hung DB backup is flipped to error promptly. #740 — Standardize tag casing showed "0 groups to change" the instant it was clicked: onNormCommit overwrote the preview with zeros. Keep the real preview visible and disable the button while queued; backend apply was already correct. Tests: fake subprocess.Popen alongside run; bounded-kill fail-fast; local-temp target; per-kind stall sweep. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/backup_service.py | 67 ++++++++++++++++--- backend/app/tasks/maintenance.py | 23 +++++-- .../settings/TagMaintenanceCard.vue | 8 ++- tests/test_backup_service.py | 65 ++++++++++++++++-- tests/test_tasks_backup.py | 52 +++++++++++++- 5 files changed, 188 insertions(+), 27 deletions(-) diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py index cc177b9..caa7cfd 100644 --- a/backend/app/services/backup_service.py +++ b/backend/app/services/backup_service.py @@ -15,7 +15,10 @@ lifecycle + soft/hard time limits + retention bookkeeping. from __future__ import annotations import json +import os +import shutil import subprocess +import tempfile from datetime import UTC, datetime from pathlib import Path @@ -26,6 +29,35 @@ _BACKUPS_DIRNAME = "_backups" # blocking syscall ignores that signal. These bound the worst case. _DB_SUBPROCESS_TIMEOUT_S = 12 * 60 # 12 min (Celery soft is 10 min) _IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60 # 7 hr (Celery soft is 6 hr) +# Grace after SIGKILL to reap the child. If it can't be reaped in this window +# (an uninterruptible NFS D-state — the failure mode that wedged the +# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we +# STOP waiting and fail fast, freeing the worker slot. The orphan is reaped by +# the OS once its blocking syscall clears. +_KILL_REAP_GRACE_S = 10 + + +def _run_bounded(cmd: list[str], timeout: int) -> None: + """subprocess.run(check=True, timeout) whose reaper can't itself hang. + + subprocess.run's timeout path SIGKILLs the child then blocks in wait() to + reap it — but a process stuck in uninterruptible I/O (NFS) can't be reaped, + so wait() blocks for hours. Here we bound the post-kill reap and re-raise + TimeoutExpired regardless, so the caller fails fast instead of wedging.""" + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.communicate(timeout=_KILL_REAP_GRACE_S) + except subprocess.TimeoutExpired: + pass # unkillable (D-state) — abandon the reap, fail fast + raise + if proc.returncode != 0: + raise subprocess.CalledProcessError( + proc.returncode, cmd, output=out, stderr=err + ) def _libpq_url(sa_url: str) -> str: @@ -84,14 +116,25 @@ def backup_db( ts = _now_ts() out_dir = _backups_dir(images_root) sql_path = out_dir / f"fc_db_{ts}.sql" - subprocess.run( - [ - "pg_dump", "--no-owner", "--no-acl", - "-f", str(sql_path), _libpq_url(db_url), - ], - capture_output=True, check=True, - timeout=_DB_SUBPROCESS_TIMEOUT_S, - ) + # Dump to LOCAL disk first, then move the finished file to the (NFS) backups + # dir. pg_dump's long phase is then a DB-socket wait + local writes — both + # killable — instead of an NFS write that can hang uninterruptibly. Only the + # final move touches NFS, and it's a bounded single-file step. + fd, tmp_name = tempfile.mkstemp(prefix="fc_db_", suffix=".sql") + os.close(fd) + tmp_path = Path(tmp_name) + try: + _run_bounded( + [ + "pg_dump", "--no-owner", "--no-acl", + "-f", str(tmp_path), _libpq_url(db_url), + ], + _DB_SUBPROCESS_TIMEOUT_S, + ) + shutil.move(str(tmp_path), str(sql_path)) + finally: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) manifest_path = _write_manifest( out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by, artifact_path=sql_path, @@ -114,15 +157,17 @@ def backup_images( ts = _now_ts() out_dir = _backups_dir(images_root) tar_path = out_dir / f"fc_images_{ts}.tar.zst" - subprocess.run( + # No local-temp here (the archive is hundreds of GB — it can't stage in + # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast + # rather than holding the lane for hours. + _run_bounded( [ "tar", "--zstd", "-cf", str(tar_path), "-C", str(images_root.parent), images_root.name, f"--exclude={images_root.name}/_backups", f"--exclude={images_root.name}/_quarantine", ], - capture_output=True, check=True, - timeout=_IMAGES_SUBPROCESS_TIMEOUT_S, + _IMAGES_SUBPROCESS_TIMEOUT_S, ) manifest_path = _write_manifest( out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by, diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 807d0e6..7e81786 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -83,8 +83,13 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days # small buffer) so the sweep never flags in-flight work. # # Backups: images backup has time_limit=23400s (6.5h). 7h covers it -# with a 30-min buffer; db backup at 12 min hard limit fits trivially. +# with a 30-min buffer. BACKUP_STALL_THRESHOLD_MINUTES = 7 * 60 +# DB backup/restore is seconds-to-minutes (35-min hard limit). It must NOT share +# the images' 7h window — a DB backup wedged on NFS would otherwise sit "running" +# for 7 hours holding the concurrency-1 maintenance_long lane (operator-flagged +# 2026-06-07). 40 min gives a small buffer over the hard limit. +BACKUP_DB_STALL_THRESHOLD_MINUTES = 40 # Library audit: scan_library_for_rule has time_limit=7500s (2h5m). # 2h15m gives a 10-min buffer. LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135 @@ -619,16 +624,20 @@ def recover_stalled_backup_runs() -> int: """ SessionLocal = _sync_session_factory() now = datetime.now(UTC) - cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES) - msg = ( - f"stranded by recovery sweep (no terminal status after " - f"{BACKUP_STALL_THRESHOLD_MINUTES // 60}h)" - ) + db_cutoff = now - timedelta(minutes=BACKUP_DB_STALL_THRESHOLD_MINUTES) + slow_cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES) + msg = "stranded by recovery sweep (no terminal status within the stall window)" with SessionLocal() as session: result = session.execute( update(BackupRun) .where(BackupRun.status.in_(["running", "restoring"])) - .where(BackupRun.started_at < cutoff) + # db backups/restores are fast (40-min window); images run hours (7h). + .where( + or_( + and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff), + and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff), + ) + ) .values(status="error", finished_at=now, error=msg) ) session.commit() diff --git a/frontend/src/components/settings/TagMaintenanceCard.vue b/frontend/src/components/settings/TagMaintenanceCard.vue index b1d996b..2122c58 100644 --- a/frontend/src/components/settings/TagMaintenanceCard.vue +++ b/frontend/src/components/settings/TagMaintenanceCard.vue @@ -181,7 +181,7 @@ Standardize {{ normPreview.total_changes }} tag group(s) @@ -291,7 +291,11 @@ async function onNormCommit() { // confirm it's queued; the operator can re-run Preview later to verify. await store.normalizeTags({ dryRun: false }) normResult.value = 'queued' - normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] } + // Keep the preview showing what was QUEUED — do NOT zero it out. Overwriting + // it with a zeroed object made the card read "0 tag groups to change" the + // instant Standardize was clicked, which looked like nothing happened + // (operator-flagged 2026-06-07). The button disables on `queued`; re-running + // Preview later reflects the real remaining count as the task applies. } finally { normCommitting.value = false } diff --git a/tests/test_backup_service.py b/tests/test_backup_service.py index 519b23c..46b1ffc 100644 --- a/tests/test_backup_service.py +++ b/tests/test_backup_service.py @@ -5,6 +5,7 @@ without external binaries. The real subprocess behavior is exercised implicitly via the Celery task tests in test_tasks_backup.py. """ import json +import subprocess from pathlib import Path import pytest @@ -16,9 +17,9 @@ pytestmark = pytest.mark.integration @pytest.fixture def fake_subprocess(monkeypatch): - """Replace subprocess.run with a fake that writes a sentinel to - the target path (for pg_dump's -f, for tar's -cf). Captures all - calls in a list.""" + """Fake both subprocess.run (restore path) AND subprocess.Popen (the + bounded-kill backup path) so tests run without external binaries. Each + writes the target sentinel and records the cmd in a shared list.""" calls = [] class _FakeProc: @@ -26,17 +27,33 @@ def fake_subprocess(monkeypatch): stdout = b"" stderr = b"" - def _fake_run(cmd, **kwargs): - calls.append(list(cmd)) + def _write_sentinel(cmd): if cmd[0] == "pg_dump": i = cmd.index("-f") Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n") elif cmd[0] == "tar" and "-cf" in cmd: i = cmd.index("-cf") Path(cmd[i + 1]).write_bytes(b"fake tar payload") + + def _fake_run(cmd, **kwargs): + calls.append(list(cmd)) + _write_sentinel(cmd) return _FakeProc() + class _FakePopen: + def __init__(self, cmd, **kwargs): + calls.append(list(cmd)) + self.returncode = 0 + _write_sentinel(cmd) + + def communicate(self, timeout=None): + return (b"", b"") + + def kill(self): + self.returncode = -9 + monkeypatch.setattr("subprocess.run", _fake_run) + monkeypatch.setattr("subprocess.Popen", _FakePopen) return calls @@ -198,3 +215,41 @@ def test_backups_dir_created_on_first_use(tmp_path): d = backup_service._backups_dir(tmp_path) assert d.is_dir() assert d.name == "_backups" + + +# --- bounded-kill + local-temp (FC #739) ----------------------------- + + +def test_backup_db_dumps_to_local_temp_not_nfs_backups_dir(tmp_path, fake_subprocess): + """pg_dump must target a LOCAL temp path, not the (NFS) _backups dir — + so its long phase can't hang uninterruptibly on an NFS write.""" + backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path) + cmd = fake_subprocess[0] + dump_target = cmd[cmd.index("-f") + 1] + assert "_backups" not in dump_target + # The finished file still ends up in _backups (moved there). + result = backup_service.backup_db( + db_url="postgresql://u@h/d", images_root=tmp_path, + ) + assert "_backups" in result["sql_path"] + + +def test_run_bounded_fails_fast_when_unkillable(monkeypatch): + """A child stuck in D-state (communicate keeps timing out even after kill) + must NOT block the reaper — _run_bounded kills then re-raises promptly.""" + killed = {"n": 0} + + class _Hang: + def __init__(self, cmd, **kwargs): + pass + + def communicate(self, timeout=None): + raise subprocess.TimeoutExpired(cmd="x", timeout=timeout or 0) + + def kill(self): + killed["n"] += 1 + + monkeypatch.setattr("subprocess.Popen", _Hang) + with pytest.raises(subprocess.TimeoutExpired): + backup_service._run_bounded(["pg_dump"], 1) + assert killed["n"] == 1 diff --git a/tests/test_tasks_backup.py b/tests/test_tasks_backup.py index 4f5ace7..4c285e3 100644 --- a/tests/test_tasks_backup.py +++ b/tests/test_tasks_backup.py @@ -34,16 +34,32 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path): stdout = b"" stderr = b"" - def _fake_run(cmd, **kwargs): + def _sentinel(cmd): if cmd[0] == "pg_dump": i = cmd.index("-f") Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n") elif cmd[0] == "tar" and "-cf" in cmd: i = cmd.index("-cf") Path(cmd[i + 1]).write_bytes(b"fake tar payload") + + def _fake_run(cmd, **kwargs): + _sentinel(cmd) return _FakeProc() + class _FakePopen: + def __init__(self, cmd, **kwargs): + self.returncode = 0 + _sentinel(cmd) + + def communicate(self, timeout=None): + return (b"", b"") + + def kill(self): + self.returncode = -9 + + # backup_db/backup_images go through _run_bounded (Popen); restore via run. monkeypatch.setattr("subprocess.run", _fake_run) + monkeypatch.setattr("subprocess.Popen", _FakePopen) def _seed_backup(db_sync, *, kind, status, started_at, tag=None, @@ -89,7 +105,8 @@ async def test_backup_db_task_records_failure_on_subprocess_error(db_sync, monke def _boom(*a, **kw): raise RuntimeError("synthetic pg_dump fail") - monkeypatch.setattr("subprocess.run", _boom) + # backup_db dumps via _run_bounded → subprocess.Popen. + monkeypatch.setattr("subprocess.Popen", _boom) with pytest.raises(RuntimeError): backup_db_task.delay().get() @@ -237,6 +254,37 @@ def test_prune_backups_never_deletes_running_or_restoring(db_sync): assert set(statuses) == {"running", "restoring"} +# --- recover_stalled_backup_runs (per-kind threshold, FC #739) ------- + + +def test_stall_sweep_flips_db_fast_but_spares_running_images(db_sync): + from backend.app.tasks.maintenance import recover_stalled_backup_runs + + now = datetime.now(UTC) + # A db backup stuck 50 min → past the 40-min db window → flipped to error. + db_id = _seed_backup( + db_sync, kind="db", status="running", + started_at=now - timedelta(minutes=50), + ) + # An images backup running 50 min is still well under the 7h window → spared. + img_id = _seed_backup( + db_sync, kind="images", status="running", + started_at=now - timedelta(minutes=50), + ) + db_sync.commit() + + recover_stalled_backup_runs.apply().get() + + db_status = db_sync.execute( + select(BackupRun.status).where(BackupRun.id == db_id) + ).scalar_one() + img_status = db_sync.execute( + select(BackupRun.status).where(BackupRun.id == img_id) + ).scalar_one() + assert db_status == "error" + assert img_status == "running" + + # --- backup_db_nightly ---------------------------------------------- -- 2.52.0