diff --git a/alembic/versions/0040_series_chapters.py b/alembic/versions/0040_series_chapters.py new file mode 100644 index 0000000..a0808df --- /dev/null +++ b/alembic/versions/0040_series_chapters.py @@ -0,0 +1,108 @@ +"""series chapters: chapter layer over series_page (FC-6.1) + +Revision ID: 0040 +Revises: 0039 +Create Date: 2026-06-07 + +A series (Tag kind='series') gains an ordered chapter layer. Reading order +becomes (series_chapter.chapter_number, series_page.page_number). Every existing +series is backfilled into a single auto-chapter (chapter_number=1) holding its +current flat pages, so no data is lost and the old flat ordering is preserved. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0040" +down_revision: Union[str, None] = "0039" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_chapter", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("chapter_number", sa.Integer, nullable=False), + sa.Column("title", sa.Text, nullable=True), + sa.Column( + "is_placeholder", sa.Boolean, nullable=False, server_default="false" + ), + sa.Column("stated_page_start", sa.Integer, nullable=True), + sa.Column("stated_page_end", sa.Integer, nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + ) + op.create_index( + "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"] + ) + + # New columns on series_page; chapter_id starts nullable so we can backfill. + op.add_column( + "series_page", sa.Column("chapter_id", sa.Integer, nullable=True) + ) + op.add_column( + "series_page", sa.Column("stated_page", sa.Integer, nullable=True) + ) + + conn = op.get_bind() + # One auto-chapter per existing series (any series_tag_id present in pages). + conn.execute( + sa.text( + "INSERT INTO series_chapter " + "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) " + "SELECT DISTINCT series_tag_id, 1, false, now(), now() " + "FROM series_page" + ) + ) + # Point every existing page at its series' auto-chapter. + conn.execute( + sa.text( + "UPDATE series_page sp " + "SET chapter_id = sc.id " + "FROM series_chapter sc " + "WHERE sc.series_tag_id = sp.series_tag_id" + ) + ) + + # Now lock chapter_id down: NOT NULL + FK (cascade) + index. + op.alter_column("series_page", "chapter_id", nullable=False) + op.create_foreign_key( + "fk_series_page_chapter_id", + "series_page", + "series_chapter", + ["chapter_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + "ix_series_page_chapter_id", "series_page", ["chapter_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_series_page_chapter_id", table_name="series_page") + op.drop_constraint( + "fk_series_page_chapter_id", "series_page", type_="foreignkey" + ) + op.drop_column("series_page", "stated_page") + op.drop_column("series_page", "chapter_id") + op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter") + op.drop_table("series_chapter") diff --git a/alembic/versions/0041_series_suggestions.py b/alembic/versions/0041_series_suggestions.py new file mode 100644 index 0000000..51b690a --- /dev/null +++ b/alembic/versions/0041_series_suggestions.py @@ -0,0 +1,98 @@ +"""series suggestions: assisted-continuation matcher (FC-6.3) + +Revision ID: 0041 +Revises: 0040 +Create Date: 2026-06-07 + +A confirm-only queue of "this post may continue this series" hints, plus two +import_settings knobs (enable + score threshold) for the matcher. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0041" +down_revision: Union[str, None] = "0040" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_suggestion", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "post_id", + sa.Integer, + sa.ForeignKey("post.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("score", sa.Float, nullable=False), + sa.Column("signals", sa.JSON, nullable=True), + sa.Column( + "status", sa.String(16), nullable=False, server_default="pending" + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + op.create_index( + "ix_series_suggestion_post_id", "series_suggestion", ["post_id"] + ) + op.create_index( + "ix_series_suggestion_series_tag_id", + "series_suggestion", + ["series_tag_id"], + ) + op.create_index( + "ix_series_suggestion_status", "series_suggestion", ["status"] + ) + + op.add_column( + "import_settings", + sa.Column( + "series_suggest_enabled", + sa.Boolean, + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "series_suggest_threshold", + sa.Float, + nullable=False, + server_default="0.5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "series_suggest_threshold") + op.drop_column("import_settings", "series_suggest_enabled") + op.drop_index("ix_series_suggestion_status", table_name="series_suggestion") + op.drop_index( + "ix_series_suggestion_series_tag_id", table_name="series_suggestion" + ) + op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion") + op.drop_table("series_suggestion") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index fa97bda..7d69fbf 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -25,6 +25,8 @@ _EDITABLE_FIELDS = ( "download_schedule_default_seconds", "download_event_retention_days", "download_failure_warning_threshold", + "series_suggest_enabled", + "series_suggest_threshold", ) @@ -46,6 +48,8 @@ async def get_import_settings(): "download_schedule_default_seconds": row.download_schedule_default_seconds, "download_event_retention_days": row.download_event_retention_days, "download_failure_warning_threshold": row.download_failure_warning_threshold, + "series_suggest_enabled": row.series_suggest_enabled, + "series_suggest_threshold": row.series_suggest_threshold, }) @@ -96,6 +100,19 @@ async def update_import_settings(): if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100: return _bad_int("download_failure_warning_threshold", 1, 100) + if "series_suggest_enabled" in body and not isinstance( + body["series_suggest_enabled"], bool + ): + return jsonify( + {"error": "series_suggest_enabled must be a boolean"} + ), 400 + if "series_suggest_threshold" in body: + v = body["series_suggest_threshold"] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: + return jsonify( + {"error": "series_suggest_threshold must be a number in [0, 1]"} + ), 400 + async with get_session() as session: row = await ImportSettings.load(session) for field in _EDITABLE_FIELDS: diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 288ade6..918fa42 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -8,6 +8,7 @@ from ..extensions import get_session from ..models import Tag, TagKind from ..models.tag_allowlist import TagAllowlist from ..services.bulk_tag_service import BulkTagService +from ..services.series_match_service import SeriesMatchService from ..services.series_service import SeriesError, SeriesService from ..services.tag_directory_service import TagDirectoryService from ..services.tag_service import ( @@ -368,6 +369,31 @@ def _series_err(exc: SeriesError): return jsonify({"error": msg}), status +def _opt_int(body, key: str): + """(value, error) — value is None when absent, error is (json, status).""" + if not body or body.get(key) is None: + return None, None + try: + return int(body[key]), None + except (TypeError, ValueError): + return None, (jsonify({"error": f"{key} must be an integer"}), 400) + + +def _parse_int_list(body, key: str, *, max_ids: int = 500): + """(list, error) for a required list of ints under `key`.""" + if not body or key not in body: + return None, (jsonify({"error": f"{key} required"}), 400) + raw = body[key] + if not isinstance(raw, list) or not raw: + return None, (jsonify({"error": f"{key} must be a non-empty list"}), 400) + if len(raw) > max_ids: + return None, (jsonify({"error": f"too many ids (max {max_ids})"}), 400) + try: + return [int(x) for x in raw], None + except (TypeError, ValueError): + return None, (jsonify({"error": f"{key} must be integers"}), 400) + + @tags_bp.route("/series//pages", methods=["GET"]) async def series_pages(tag_id: int): async with get_session() as session: @@ -384,9 +410,14 @@ async def series_add(tag_id: int): ids, err = _parse_bulk_ids(body, max_ids=500) if err: return err + chapter_id, cerr = _opt_int(body, "chapter_id") + if cerr: + return cerr async with get_session() as session: try: - n = await SeriesService(session).add_images(tag_id, ids) + n = await SeriesService(session).add_images( + tag_id, ids, chapter_id=chapter_id + ) except SeriesError as exc: return _series_err(exc) await session.commit() @@ -439,3 +470,227 @@ async def series_cover(tag_id: int): return _series_err(exc) await session.commit() return jsonify({"ok": True}) + + +# ---- chapters (FC-6.1) ---------------------------------------------------- + + +@tags_bp.route("/series//chapters", methods=["POST"]) +async def series_chapter_create(tag_id: int): + body = await request.get_json() or {} + title = body.get("title") + if title is not None and not isinstance(title, str): + return jsonify({"error": "title must be a string"}), 400 + is_placeholder = bool(body.get("is_placeholder", False)) + start, serr = _opt_int(body, "stated_page_start") + if serr: + return serr + end, eerr = _opt_int(body, "stated_page_end") + if eerr: + return eerr + async with get_session() as session: + try: + ch = await SeriesService(session).create_chapter( + tag_id, + title=title, + is_placeholder=is_placeholder, + stated_page_start=start, + stated_page_end=end, + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(ch) + + +@tags_bp.route("/series//chapters/reorder", methods=["POST"]) +async def series_chapter_reorder(tag_id: int): + body = await request.get_json() + ids, err = _parse_int_list(body, "chapter_ids") + if err: + return err + async with get_session() as session: + try: + await SeriesService(session).reorder_chapters(tag_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters/", methods=["PATCH"] +) +async def series_chapter_update(tag_id: int, chapter_id: int): + body = await request.get_json() or {} + kwargs: dict = {} + if "title" in body: + if body["title"] is not None and not isinstance(body["title"], str): + return jsonify({"error": "title must be a string"}), 400 + kwargs.update(set_title=True, title=body["title"]) + if "stated_page_start" in body: + start, serr = _opt_int(body, "stated_page_start") + if serr: + return serr + kwargs.update(set_start=True, stated_page_start=start) + if "stated_page_end" in body: + end, eerr = _opt_int(body, "stated_page_end") + if eerr: + return eerr + kwargs.update(set_end=True, stated_page_end=end) + async with get_session() as session: + try: + await SeriesService(session).update_chapter( + tag_id, chapter_id, **kwargs + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters/", methods=["DELETE"] +) +async def series_chapter_delete(tag_id: int, chapter_id: int): + async with get_session() as session: + try: + await SeriesService(session).delete_chapter(tag_id, chapter_id) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route( + "/series//chapters//merge", methods=["POST"] +) +async def series_chapter_merge(tag_id: int, chapter_id: int): + body = await request.get_json() + target, terr = _opt_int(body, "target_chapter_id") + if terr: + return terr + if target is None: + return jsonify({"error": "target_chapter_id required"}), 400 + async with get_session() as session: + try: + moved = await SeriesService(session).merge_chapter( + tag_id, chapter_id, target + ) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"moved_count": moved}) + + +@tags_bp.route( + "/series//chapters//reorder", methods=["POST"] +) +async def series_chapter_reorder_pages(tag_id: int, chapter_id: int): + body = await request.get_json() + ids, err = _parse_bulk_ids(body, max_ids=500) + if err: + return err + async with get_session() as session: + try: + await SeriesService(session).reorder_pages(tag_id, chapter_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +# ---- browse list + post→series flows (FC-6.2) ----------------------------- + + +@tags_bp.route("/series", methods=["GET"]) +async def series_list(): + args = request.args + sort = args.get("sort", "recent") + if sort not in ("recent", "name", "size"): + return jsonify({"error": "sort must be recent|name|size"}), 400 + artist_id = None + if args.get("artist_id") is not None: + try: + artist_id = int(args["artist_id"]) + except ValueError: + return jsonify({"error": "artist_id must be an integer"}), 400 + async with get_session() as session: + rows = await SeriesService(session).list_series( + sort=sort, artist_id=artist_id + ) + return jsonify({"series": rows}) + + +@tags_bp.route("/series/from-post", methods=["POST"]) +async def series_from_post(): + body = await request.get_json() + post_id, err = _opt_int(body, "post_id") + if err: + return err + if post_id is None: + return jsonify({"error": "post_id required"}), 400 + async with get_session() as session: + try: + out = await SeriesService(session).promote_post_to_series(post_id) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(out) + + +@tags_bp.route("/series//add-post", methods=["POST"]) +async def series_add_post(tag_id: int): + body = await request.get_json() + post_id, err = _opt_int(body, "post_id") + if err: + return err + if post_id is None: + return jsonify({"error": "post_id required"}), 400 + async with get_session() as session: + try: + out = await SeriesService(session).add_post_as_chapter(tag_id, post_id) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(out) + + +# ---- suggestion queue (FC-6.3) -------------------------------------------- + + +@tags_bp.route("/series/suggestions", methods=["GET"]) +async def series_suggestions_list(): + async with get_session() as session: + rows = await SeriesMatchService(session).list_pending() + return jsonify({"suggestions": rows}) + + +@tags_bp.route("/series/suggestions//accept", methods=["POST"]) +async def series_suggestion_accept(sid: int): + async with get_session() as session: + try: + out = await SeriesMatchService(session).accept(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(out) + + +@tags_bp.route("/series/suggestions//dismiss", methods=["POST"]) +async def series_suggestion_dismiss(sid: int): + async with get_session() as session: + try: + await SeriesMatchService(session).dismiss(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route("/series/suggestions/rescan", methods=["POST"]) +async def series_suggestions_rescan(): + from ..tasks.admin import rescan_series_suggestions_task + + res = rescan_series_suggestions_task.delay() + return jsonify({"task_id": res.id}) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index b8854b1..00047f9 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -18,7 +18,9 @@ from .patreon_failed_media import PatreonFailedMedia from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment +from .series_chapter import SeriesChapter from .series_page import SeriesPage +from .series_suggestion import SeriesSuggestion from .source import Source from .tag import Tag, TagKind, image_tag from .tag_alias import TagAlias @@ -39,7 +41,9 @@ __all__ = [ "PatreonSeenMedia", "Post", "PostAttachment", + "SeriesChapter", "SeriesPage", + "SeriesSuggestion", "ImageRecord", "ImageProvenance", "Tag", diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index e5142ca..8c59fa8 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -64,6 +64,15 @@ class ImportSettings(Base): Integer, nullable=False, default=3, ) + # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is + # the weighted-score cut-off (0..1) above which a pending suggestion is made. + series_suggest_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, + ) + series_suggest_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.5, + ) + @classmethod async def load(cls, session) -> ImportSettings: """The singleton settings row (id=1), via an async session.""" diff --git a/backend/app/models/series_chapter.py b/backend/app/models/series_chapter.py new file mode 100644 index 0000000..4012174 --- /dev/null +++ b/backend/app/models/series_chapter.py @@ -0,0 +1,47 @@ +"""SeriesChapter — an ordered chapter/part within a series. + +A series IS a Tag(kind='series'); a chapter groups ordered SeriesPages under it. +Reading order is (chapter.chapter_number, series_page.page_number): chapter_number +sets the order of chapters, page_number orders pages within a chapter. + +chapter_number is an ordering key only (not unique) — reorder rewrites 1..N +wholesale, mirroring series_page.page_number, so a reorder can't transiently +collide on a unique index. + +A chapter may be a placeholder (is_placeholder=True) — a reserved empty slot for +a section the operator doesn't have yet; it holds no pages and shows as a gap in +the reader. stated_page_start/end carry the page range parsed from the source +post (FC-6.2), used to flag missing-page gaps; both are nullable when unknown. +""" + +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, Text, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesChapter(Base): + __tablename__ = "series_chapter" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + chapter_number: Mapped[int] = mapped_column(Integer, nullable=False) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + is_placeholder: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default="false" + ) + stated_page_start: Mapped[int | None] = mapped_column(Integer, nullable=True) + stated_page_end: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/models/series_page.py b/backend/app/models/series_page.py index 1ebcd7d..48384ef 100644 --- a/backend/app/models/series_page.py +++ b/backend/app/models/series_page.py @@ -1,9 +1,12 @@ """SeriesPage — ordered image membership for a series-kind Tag. -A series IS a Tag with kind='series'; series_page gives it ordered pages. -An image belongs to at most one series (UNIQUE image_id). Cover = the -lowest page_number. page_number is an ordering key only (not unique) — -reorder rewrites 1..N wholesale. +A series IS a Tag with kind='series'; series_page gives it ordered pages, +grouped into chapters (FC-6). An image belongs to at most one series +(UNIQUE image_id) ⇒ at most one chapter. Reading order is +(chapter.chapter_number, series_page.page_number): page_number orders pages +WITHIN a chapter and is an ordering key only (not unique) — reorder rewrites +1..N wholesale. stated_page carries the page number parsed from the source +post (FC-6.2), nullable when unknown. """ from datetime import datetime @@ -21,12 +24,18 @@ class SeriesPage(Base): series_tag_id: Mapped[int] = mapped_column( ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True ) + chapter_id: Mapped[int] = mapped_column( + ForeignKey("series_chapter.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) image_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, unique=True, ) page_number: Mapped[int] = mapped_column(Integer, nullable=False) + stated_page: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/series_suggestion.py b/backend/app/models/series_suggestion.py new file mode 100644 index 0000000..316483b --- /dev/null +++ b/backend/app/models/series_suggestion.py @@ -0,0 +1,55 @@ +"""SeriesSuggestion — a confirm-only "this post may continue this series" hint. + +The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted +signals and, above the configured threshold, records a pending suggestion. The +operator confirms (→ the post is added as a chapter) or dismisses it; FC never +files a post into a series on its own. status is a plain string (no Postgres +ENUM — see the check-existing-enums lesson): pending | added | dismissed. +""" + +from datetime import datetime + +from sqlalchemy import ( + JSON, + DateTime, + Float, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesSuggestion(Base): + __tablename__ = "series_suggestion" + __table_args__ = ( + UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + score: Mapped[float] = mapped_column(Float, nullable=False) + signals: Mapped[dict | None] = mapped_column(JSON, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending", index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py index cc177b9..caa7cfd 100644 --- a/backend/app/services/backup_service.py +++ b/backend/app/services/backup_service.py @@ -15,7 +15,10 @@ lifecycle + soft/hard time limits + retention bookkeeping. from __future__ import annotations import json +import os +import shutil import subprocess +import tempfile from datetime import UTC, datetime from pathlib import Path @@ -26,6 +29,35 @@ _BACKUPS_DIRNAME = "_backups" # blocking syscall ignores that signal. These bound the worst case. _DB_SUBPROCESS_TIMEOUT_S = 12 * 60 # 12 min (Celery soft is 10 min) _IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60 # 7 hr (Celery soft is 6 hr) +# Grace after SIGKILL to reap the child. If it can't be reaped in this window +# (an uninterruptible NFS D-state — the failure mode that wedged the +# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we +# STOP waiting and fail fast, freeing the worker slot. The orphan is reaped by +# the OS once its blocking syscall clears. +_KILL_REAP_GRACE_S = 10 + + +def _run_bounded(cmd: list[str], timeout: int) -> None: + """subprocess.run(check=True, timeout) whose reaper can't itself hang. + + subprocess.run's timeout path SIGKILLs the child then blocks in wait() to + reap it — but a process stuck in uninterruptible I/O (NFS) can't be reaped, + so wait() blocks for hours. Here we bound the post-kill reap and re-raise + TimeoutExpired regardless, so the caller fails fast instead of wedging.""" + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.communicate(timeout=_KILL_REAP_GRACE_S) + except subprocess.TimeoutExpired: + pass # unkillable (D-state) — abandon the reap, fail fast + raise + if proc.returncode != 0: + raise subprocess.CalledProcessError( + proc.returncode, cmd, output=out, stderr=err + ) def _libpq_url(sa_url: str) -> str: @@ -84,14 +116,25 @@ def backup_db( ts = _now_ts() out_dir = _backups_dir(images_root) sql_path = out_dir / f"fc_db_{ts}.sql" - subprocess.run( - [ - "pg_dump", "--no-owner", "--no-acl", - "-f", str(sql_path), _libpq_url(db_url), - ], - capture_output=True, check=True, - timeout=_DB_SUBPROCESS_TIMEOUT_S, - ) + # Dump to LOCAL disk first, then move the finished file to the (NFS) backups + # dir. pg_dump's long phase is then a DB-socket wait + local writes — both + # killable — instead of an NFS write that can hang uninterruptibly. Only the + # final move touches NFS, and it's a bounded single-file step. + fd, tmp_name = tempfile.mkstemp(prefix="fc_db_", suffix=".sql") + os.close(fd) + tmp_path = Path(tmp_name) + try: + _run_bounded( + [ + "pg_dump", "--no-owner", "--no-acl", + "-f", str(tmp_path), _libpq_url(db_url), + ], + _DB_SUBPROCESS_TIMEOUT_S, + ) + shutil.move(str(tmp_path), str(sql_path)) + finally: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) manifest_path = _write_manifest( out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by, artifact_path=sql_path, @@ -114,15 +157,17 @@ def backup_images( ts = _now_ts() out_dir = _backups_dir(images_root) tar_path = out_dir / f"fc_images_{ts}.tar.zst" - subprocess.run( + # No local-temp here (the archive is hundreds of GB — it can't stage in + # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast + # rather than holding the lane for hours. + _run_bounded( [ "tar", "--zstd", "-cf", str(tar_path), "-C", str(images_root.parent), images_root.name, f"--exclude={images_root.name}/_backups", f"--exclude={images_root.name}/_quarantine", ], - capture_output=True, check=True, - timeout=_IMAGES_SUBPROCESS_TIMEOUT_S, + _IMAGES_SUBPROCESS_TIMEOUT_S, ) manifest_path = _write_manifest( out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by, diff --git a/backend/app/services/page_number_parser.py b/backend/app/services/page_number_parser.py new file mode 100644 index 0000000..2c0c348 --- /dev/null +++ b/backend/app/services/page_number_parser.py @@ -0,0 +1,41 @@ +"""Parse a stated page number/range out of a post's title/description (FC-6.2). + +Artists often state where an installment sits in a series — "pages 9-12", +"Page 5", "[3/8]". We use that to order chapters and flag missing-page gaps. +This is best-effort: a confident match wins, otherwise we return None and the +caller falls back to capture/post-date order. Keep it conservative — a wrong +page number is worse than no page number — so matches require an explicit +page keyword (page/pg/pp) or a bracketed N/M fraction, never a bare number. + +Supported forms (case-insensitive): + range "pages 9-12", "pg 9–12", "pp. 9 - 12" -> (9, 12) + fraction "page 3 of 8", "pg 3/8", "[3/8]", "(3/8)" -> (3, 3) + single "page 5", "pg 5", "pp 5" -> (5, 5) +""" + +import re + +# Page keyword: page/pages/pg/pgs/pp/pp. (NOT a bare "p" — too many false hits.) +_KW = r"(?:pages?|pgs?|pp\.?)" +_DASH = r"[-–—]" + +_RANGE = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\s*{_DASH}\s*(\d{{1,4}})", re.I) +_OF = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\s*(?:of|/)\s*\d{{1,4}}\b", re.I) +_BRACKET = re.compile(r"[\[(]\s*(\d{1,4})\s*/\s*\d{1,4}\s*[\])]") +_SINGLE = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\b", re.I) + + +def parse_page_range(text: str | None) -> tuple[int, int] | None: + """Return (start, end) or None. start <= end; a single page yields (n, n).""" + if not text: + return None + m = _RANGE.search(text) + if m: + a, b = int(m.group(1)), int(m.group(2)) + return (a, b) if a <= b else (b, a) + for rx in (_OF, _BRACKET, _SINGLE): + m = rx.search(text) + if m: + n = int(m.group(1)) + return (n, n) + return None diff --git a/backend/app/services/series_match_service.py b/backend/app/services/series_match_service.py new file mode 100644 index 0000000..364229f --- /dev/null +++ b/backend/app/services/series_match_service.py @@ -0,0 +1,337 @@ +"""FC-6.3 — assisted continuation matcher. + +Scores a (post, candidate series) pair from several weighted signals; above the +configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the +operator accepts (post becomes a chapter) or dismisses. No single signal gates; +the score is an additive weighted sum, each signal a 0..1 strength. + +The learned "title pattern" isn't persisted — it's derived on the fly from the +post titles already in a series, so it sharpens automatically as more posts are +confirmed into the series. +""" + +import re +import time + +from sqlalchemy import and_, func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord, Post, Tag, TagKind +from ..models.series_chapter import SeriesChapter +from ..models.series_page import SeriesPage +from ..models.series_suggestion import SeriesSuggestion +from ..models.tag import image_tag +from .page_number_parser import parse_page_range +from .series_service import SeriesError, SeriesService + +# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants; +# only the on/off + threshold are operator-tunable (sensitivity is the knob +# that matters; per-signal weights are an over-tune for v1). +WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15} + +_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom) +_MAX_CANDIDATES = 50 + +# Strip installment markers so titles collapse to their stable stem. +_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I) +_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]") +_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$") + + +def normalize_title(title: str | None) -> str: + if not title: + return "" + t = _PAGE_TOKEN.sub("", title) + t = _BRACKET_NUM.sub("", t) + t = _TRAILING_NUM.sub("", t) + return " ".join(t.split()).strip().lower() + + +def _common_prefix(strings: list[str]) -> str: + strings = [s for s in strings if s] + if not strings: + return "" + pre = strings[0] + for s in strings[1:]: + i = 0 + while i < len(pre) and i < len(s) and pre[i] == s[i]: + i += 1 + pre = pre[:i] + if not pre: + break + return pre.strip() + + +def title_signal(series_titles: list[str], post_title: str | None) -> float: + """Overlap of the post title against the series' title stem (the longest + common prefix of its known titles, page/installment markers removed).""" + norm = [normalize_title(t) for t in series_titles] + norm = [t for t in norm if t] + pt = normalize_title(post_title) + if not norm or not pt: + return 0.0 + stem = _common_prefix(norm) + if len(stem) < 3: + stem = max(norm, key=len) + i = 0 + while i < len(stem) and i < len(pt) and stem[i] == pt[i]: + i += 1 + return min(1.0, i / max(len(stem), 4)) + + +def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float: + """1.0 when the post's first page continues right after the series' last + stated page; partial for a near-continuation; 0 otherwise.""" + if series_max_stated_end is None or post_start is None: + return 0.0 + diff = post_start - series_max_stated_end + if diff == 1: + return 1.0 + if 2 <= diff <= 3: + return 0.5 + return 0.0 + + +def tags_signal(shared_distinctive: int) -> float: + if shared_distinctive <= 0: + return 0.0 + return min(1.0, shared_distinctive / 3.0) + + +def weighted_score(signals: dict) -> float: + return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) + + +class SeriesMatchService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _post_image_ids(self, post_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id) + ) + ).scalars().all() + return list(rows) + + async def _series_image_ids(self, series_tag_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesPage.image_id).where( + SeriesPage.series_tag_id == series_tag_id + ) + ) + ).scalars().all() + return set(rows) + + async def _series_post_titles(self, series_tag_id: int) -> list[str]: + rows = ( + await self.session.execute( + select(Post.post_title) + .select_from(SeriesPage) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .join(Post, Post.id == ImageRecord.primary_post_id) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + Post.post_title.isnot(None), + ) + ) + .distinct() + ) + ).scalars().all() + return [t for t in rows if t] + + async def _series_max_stated_end(self, series_tag_id: int) -> int | None: + return await self.session.scalar( + select(func.max(SeriesChapter.stated_page_end)).where( + SeriesChapter.series_tag_id == series_tag_id + ) + ) + + async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]: + ids = list(image_ids) + if not ids: + return set() + rows = ( + await self.session.execute( + select(image_tag.c.tag_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where( + and_( + image_tag.c.image_record_id.in_(ids), + Tag.kind.in_(_DISTINCTIVE_KINDS), + ) + ) + .distinct() + ) + ).scalars().all() + return set(rows) + + async def _candidate_series(self, artist_id: int) -> list[int]: + """Series that already contain a page by this artist — cross-artist + series are rare, so same-artist is the candidate bound.""" + rows = ( + await self.session.execute( + select(SeriesPage.series_tag_id) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(ImageRecord.artist_id == artist_id) + .distinct() + ) + ).scalars().all() + return list(rows) + + async def _decided_series(self, post_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesSuggestion.series_tag_id).where( + and_( + SeriesSuggestion.post_id == post_id, + SeriesSuggestion.status.in_(["added", "dismissed"]), + ) + ) + ) + ).scalars().all() + return set(rows) + + async def match_post(self, post_id: int, *, threshold: float) -> int: + """Score a post against its artist's series; upsert pending suggestions + for those at/above threshold. Returns the number written.""" + post = await self.session.get(Post, post_id) + if post is None or post.artist_id is None: + return 0 + post_images = await self._post_image_ids(post_id) + if not post_images: + return 0 + post_image_set = set(post_images) + rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}") + post_start = rng[0] if rng else None + post_dtags = await self._distinctive_tags(post_images) + + decided = await self._decided_series(post_id) + candidates = await self._candidate_series(post.artist_id) + made = 0 + for sid in candidates[:_MAX_CANDIDATES]: + if sid in decided: + continue + s_images = await self._series_image_ids(sid) + if post_image_set & s_images: + continue # the post is already (partly) in this series + signals = { + "title": title_signal( + await self._series_post_titles(sid), post.post_title + ), + "artist": 1.0, # candidates are same-artist by construction + "pages": pages_signal( + await self._series_max_stated_end(sid), post_start + ), + "tags": tags_signal( + len(post_dtags & await self._distinctive_tags(s_images)) + ), + } + score = weighted_score(signals) + if score < threshold: + continue + await self.session.execute( + pg_insert(SeriesSuggestion) + .values( + post_id=post_id, series_tag_id=sid, + score=score, signals=signals, status="pending", + ) + .on_conflict_do_update( + constraint="uq_series_suggestion_post_series", + set_={"score": score, "signals": signals, "status": "pending"}, + where=SeriesSuggestion.status == "pending", + ) + ) + made += 1 + return made + + # ---- queue ops -------------------------------------------------------- + + async def list_pending(self) -> list[dict]: + rows = ( + await self.session.execute( + select( + SeriesSuggestion.id, + SeriesSuggestion.post_id, + SeriesSuggestion.series_tag_id, + SeriesSuggestion.score, + SeriesSuggestion.signals, + Post.post_title, + Post.external_post_id, + Tag.name.label("series_name"), + ) + .join(Post, Post.id == SeriesSuggestion.post_id) + .join(Tag, Tag.id == SeriesSuggestion.series_tag_id) + .where(SeriesSuggestion.status == "pending") + .order_by(SeriesSuggestion.score.desc()) + ) + ).all() + return [ + { + "id": r.id, + "post_id": r.post_id, + "series_tag_id": r.series_tag_id, + "series_name": r.series_name, + "post_title": r.post_title or f"Post {r.external_post_id}", + "score": r.score, + "signals": r.signals or {}, + } + for r in rows + ] + + async def accept(self, suggestion_id: int) -> dict: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status != "pending": + raise SeriesError(f"suggestion {suggestion_id} is already {s.status}") + out = await SeriesService(self.session).add_post_as_chapter( + s.series_tag_id, s.post_id + ) + s.status = "added" + self.session.add(s) + return out + + async def dismiss(self, suggestion_id: int) -> None: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status == "pending": + s.status = "dismissed" + self.session.add(s) + + async def rescan( + self, *, threshold: float, time_budget_seconds: float | None = None, + after_post_id: int = 0, + ) -> dict: + """Score every post (id > after_post_id) against its artist's series, + time-boxed + resumable like the other long maintenance sweeps.""" + post_ids = ( + await self.session.execute( + select(Post.id) + .where(Post.id > after_post_id) + .order_by(Post.id.asc()) + ) + ).scalars().all() + summary = { + "scanned": 0, "suggested": 0, + "partial": False, "resume_after_id": after_post_id, + } + start = time.monotonic() + for pid in post_ids: + summary["scanned"] += 1 + summary["resume_after_id"] = pid + summary["suggested"] += await self.match_post(pid, threshold=threshold) + await self.session.commit() # commit per post so progress survives + if ( + time_budget_seconds is not None + and time.monotonic() - start >= time_budget_seconds + ): + summary["partial"] = True + break + else: + summary["partial"] = False + return summary diff --git a/backend/app/services/series_service.py b/backend/app/services/series_service.py index 361caa5..7c943a1 100644 --- a/backend/app/services/series_service.py +++ b/backend/app/services/series_service.py @@ -1,4 +1,8 @@ -"""Series = a series-kind Tag + ordered series_page membership. +"""Series = a series-kind Tag + ordered chapters, each holding ordered pages. + +Reading order is (series_chapter.chapter_number, series_page.page_number). A +chapter may be a placeholder (no pages) reserving a slot. An image lives in at +most one series ⇒ one chapter (series_page.image_id is UNIQUE). All mutations are Core/set-based and run in the request transaction. """ @@ -7,9 +11,11 @@ from sqlalchemy import and_, func, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession -from ..models import ImageRecord, Tag, TagKind +from ..models import Artist, ImageRecord, Post, Tag, TagKind +from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage from .gallery_service import thumbnail_url +from .page_number_parser import parse_page_range class SeriesError(ValueError): @@ -21,6 +27,8 @@ class SeriesService: def __init__(self, session: AsyncSession): self.session = session + # ---- guards / helpers ------------------------------------------------- + async def _require_series(self, series_tag_id: int) -> Tag: tag = await self.session.get(Tag, series_tag_id) if tag is None: @@ -32,7 +40,7 @@ class SeriesService: @staticmethod def _clean_ids(ids: list[int]) -> list[int]: if not ids: - raise SeriesError("image_ids must be a non-empty list") + raise SeriesError("ids must be a non-empty list") if len(ids) > 500: raise SeriesError("selection too large (max 500)") seen: set[int] = set() @@ -43,51 +51,172 @@ class SeriesService: out.append(int(x)) return out - async def _member_order(self, series_tag_id: int) -> list[int]: + async def _require_chapter(self, series_tag_id: int, chapter_id: int): + row = ( + await self.session.execute( + select(SeriesChapter).where( + and_( + SeriesChapter.id == chapter_id, + SeriesChapter.series_tag_id == series_tag_id, + ) + ) + ) + ).scalar_one_or_none() + if row is None: + raise SeriesError( + f"chapter {chapter_id} is not in series {series_tag_id}" + ) + return row + + async def _ensure_default_chapter(self, series_tag_id: int) -> int: + """Lowest-numbered chapter of the series, creating a first chapter if + the series has none (a fresh series, or the legacy single-chapter path).""" + existing = await self.session.scalar( + select(SeriesChapter.id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by(SeriesChapter.chapter_number.asc()) + .limit(1) + ) + if existing is not None: + return existing + res = await self.session.execute( + pg_insert(SeriesChapter) + .values(series_tag_id=series_tag_id, chapter_number=1) + .returning(SeriesChapter.id) + ) + return res.scalar_one() + + async def _chapter_page_order(self, chapter_id: int) -> list[int]: rows = ( await self.session.execute( select(SeriesPage.image_id) - .where(SeriesPage.series_tag_id == series_tag_id) + .where(SeriesPage.chapter_id == chapter_id) .order_by(SeriesPage.page_number.asc()) ) ).scalars().all() return list(rows) + async def _chapter_order(self, series_tag_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(SeriesChapter.id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by(SeriesChapter.chapter_number.asc()) + ) + ).scalars().all() + return list(rows) + + # ---- read ------------------------------------------------------------- + + @staticmethod + def _gaps(chapters: list[dict]) -> list[dict]: + """Missing-page gaps between consecutive chapters with stated ranges.""" + out: list[dict] = [] + prev = None + for ch in chapters: + start = ch["stated_page_start"] + if ( + prev is not None + and prev["stated_page_end"] is not None + and start is not None + and start > prev["stated_page_end"] + 1 + ): + out.append( + { + "after_chapter_id": prev["id"], + "start": prev["stated_page_end"] + 1, + "end": start - 1, + } + ) + prev = ch + return out + async def list_pages(self, series_tag_id: int) -> dict: tag = await self._require_series(series_tag_id) rows = ( await self.session.execute( select( + SeriesChapter.id.label("chapter_id"), + SeriesChapter.chapter_number, + SeriesChapter.title, + SeriesChapter.is_placeholder, + SeriesChapter.stated_page_start, + SeriesChapter.stated_page_end, SeriesPage.image_id, SeriesPage.page_number, + SeriesPage.stated_page, ImageRecord.sha256, ImageRecord.mime, ImageRecord.path, ImageRecord.thumbnail_path, ) - .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) - .where(SeriesPage.series_tag_id == series_tag_id) - .order_by(SeriesPage.page_number.asc()) + .select_from(SeriesChapter) + .outerjoin(SeriesPage, SeriesPage.chapter_id == SeriesChapter.id) + .outerjoin(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by( + SeriesChapter.chapter_number.asc(), + SeriesPage.page_number.asc(), + ) ) ).all() + + chapters: list[dict] = [] + flat: list[dict] = [] + by_id: dict[int, dict] = {} + for r in rows: + ch = by_id.get(r.chapter_id) + if ch is None: + ch = { + "id": r.chapter_id, + "chapter_number": r.chapter_number, + "title": r.title, + "is_placeholder": r.is_placeholder, + "stated_page_start": r.stated_page_start, + "stated_page_end": r.stated_page_end, + "pages": [], + } + by_id[r.chapter_id] = ch + chapters.append(ch) + if r.image_id is None: + continue # placeholder / empty chapter + page = { + "image_id": r.image_id, + "chapter_id": r.chapter_id, + "page_number": r.page_number, + "stated_page": r.stated_page, + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), + "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", + } + ch["pages"].append(page) + flat.append(page) + return { "series": {"id": tag.id, "name": tag.name}, - "pages": [ - { - "image_id": r.image_id, - "page_number": r.page_number, - "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), - "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", - } - for r in rows - ], + "chapters": chapters, + "pages": flat, # back-compat: flat reading order across chapters + "gaps": self._gaps(chapters), } + # ---- pages ------------------------------------------------------------ + async def add_images( - self, series_tag_id: int, image_ids: list[int] + self, + series_tag_id: int, + image_ids: list[int], + chapter_id: int | None = None, + stated_pages: dict[int, int] | None = None, ) -> int: + """Append images as pages of a chapter (the series' default chapter when + chapter_id is None). Images already in THIS series are left untouched; + images in another series are moved here (image_id is UNIQUE).""" await self._require_series(series_tag_id) ids = self._clean_ids(image_ids) + if chapter_id is None: + chapter_id = await self._ensure_default_chapter(series_tag_id) + else: + await self._require_chapter(series_tag_id, chapter_id) + existing = dict( ( await self.session.execute( @@ -96,29 +225,30 @@ class SeriesService: ) ).all() ) - # Already in THIS series → leave untouched (no churn). to_add = [i for i in ids if existing.get(i) != series_tag_id] if not to_add: return 0 - # Remove any prior membership for the to_add images (the "move"). + # Move: drop any prior membership for these images (UNIQUE image_id). await self.session.execute( - SeriesPage.__table__.delete().where( - SeriesPage.image_id.in_(to_add) - ) + SeriesPage.__table__.delete().where(SeriesPage.image_id.in_(to_add)) ) max_pn = ( await self.session.scalar( - select(func.coalesce(func.max(SeriesPage.page_number), 0)) - .where(SeriesPage.series_tag_id == series_tag_id) + select(func.coalesce(func.max(SeriesPage.page_number), 0)).where( + SeriesPage.chapter_id == chapter_id + ) ) ) or 0 + sp = stated_pages or {} await self.session.execute( pg_insert(SeriesPage).values( [ { "series_tag_id": series_tag_id, + "chapter_id": chapter_id, "image_id": iid, "page_number": max_pn + offset, + "stated_page": sp.get(iid), } for offset, iid in enumerate(to_add, start=1) ] @@ -141,32 +271,465 @@ class SeriesService: ) return res.rowcount or 0 - async def reorder( - self, series_tag_id: int, ordered_image_ids: list[int] + async def reorder_pages( + self, series_tag_id: int, chapter_id: int, ordered_image_ids: list[int] ) -> None: await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) ordered = self._clean_ids(ordered_image_ids) - current = set(await self._member_order(series_tag_id)) + current = set(await self._chapter_page_order(chapter_id)) if set(ordered) != current or len(ordered) != len(current): raise SeriesError( - "ordered image_ids must exactly match series membership" + "ordered image_ids must exactly match the chapter's pages" ) for idx, iid in enumerate(ordered, start=1): await self.session.execute( update(SeriesPage) .where( and_( - SeriesPage.series_tag_id == series_tag_id, + SeriesPage.chapter_id == chapter_id, SeriesPage.image_id == iid, ) ) .values(page_number=idx) ) - async def set_cover(self, series_tag_id: int, image_id: int) -> None: + async def reorder( + self, series_tag_id: int, ordered_image_ids: list[int] + ) -> None: + """Legacy series-wide reorder — reorders the default chapter. Valid only + for single-chapter series (the only shape the pre-chapter UI produced); + multi-chapter series must use reorder_pages with an explicit chapter.""" await self._require_series(series_tag_id) - order = await self._member_order(series_tag_id) - if image_id not in order: + chapters = await self._chapter_order(series_tag_id) + if len(chapters) > 1: + raise SeriesError( + "series has multiple chapters; reorder a specific chapter" + ) + chapter_id = ( + chapters[0] + if chapters + else await self._ensure_default_chapter(series_tag_id) + ) + await self.reorder_pages(series_tag_id, chapter_id, ordered_image_ids) + + # ---- chapters --------------------------------------------------------- + + async def create_chapter( + self, + series_tag_id: int, + *, + title: str | None = None, + is_placeholder: bool = False, + stated_page_start: int | None = None, + stated_page_end: int | None = None, + ) -> dict: + await self._require_series(series_tag_id) + max_cn = ( + await self.session.scalar( + select( + func.coalesce(func.max(SeriesChapter.chapter_number), 0) + ).where(SeriesChapter.series_tag_id == series_tag_id) + ) + ) or 0 + res = await self.session.execute( + pg_insert(SeriesChapter) + .values( + series_tag_id=series_tag_id, + chapter_number=max_cn + 1, + title=title, + is_placeholder=is_placeholder, + stated_page_start=stated_page_start, + stated_page_end=stated_page_end, + ) + .returning(SeriesChapter.id, SeriesChapter.chapter_number) + ) + row = res.one() + return {"id": row.id, "chapter_number": row.chapter_number} + + async def update_chapter( + self, + series_tag_id: int, + chapter_id: int, + *, + title: str | None = None, + stated_page_start: int | None = None, + stated_page_end: int | None = None, + set_title: bool = False, + set_start: bool = False, + set_end: bool = False, + ) -> None: + """Partial chapter edit. The set_* flags say which fields to write (so + None can be written explicitly, e.g. clearing a stated page).""" + await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) + values: dict = {} + if set_title: + values["title"] = title + if set_start: + values["stated_page_start"] = stated_page_start + if set_end: + values["stated_page_end"] = stated_page_end + if not values: + return + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == chapter_id) + .values(**values) + ) + + async def _renumber_chapters(self, series_tag_id: int) -> None: + for idx, cid in enumerate( + await self._chapter_order(series_tag_id), start=1 + ): + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == cid) + .values(chapter_number=idx) + ) + + async def reorder_chapters( + self, series_tag_id: int, ordered_chapter_ids: list[int] + ) -> None: + await self._require_series(series_tag_id) + ordered = self._clean_ids(ordered_chapter_ids) + current = set(await self._chapter_order(series_tag_id)) + if set(ordered) != current or len(ordered) != len(current): + raise SeriesError( + "ordered chapter_ids must exactly match the series' chapters" + ) + for idx, cid in enumerate(ordered, start=1): + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.id == cid) + .values(chapter_number=idx) + ) + + async def delete_chapter(self, series_tag_id: int, chapter_id: int) -> None: + await self._require_series(series_tag_id) + await self._require_chapter(series_tag_id, chapter_id) + # Pages cascade-delete with the chapter (FK ondelete=CASCADE). + await self.session.execute( + SeriesChapter.__table__.delete().where( + SeriesChapter.id == chapter_id + ) + ) + await self._renumber_chapters(series_tag_id) + + async def merge_chapter( + self, series_tag_id: int, source_chapter_id: int, target_chapter_id: int + ) -> int: + """Move source chapter's pages onto the end of the target chapter, then + delete the now-empty source chapter. Returns the number of pages moved.""" + await self._require_series(series_tag_id) + if source_chapter_id == target_chapter_id: + raise SeriesError("cannot merge a chapter into itself") + await self._require_chapter(series_tag_id, source_chapter_id) + await self._require_chapter(series_tag_id, target_chapter_id) + moving = await self._chapter_page_order(source_chapter_id) + max_pn = ( + await self.session.scalar( + select(func.coalesce(func.max(SeriesPage.page_number), 0)).where( + SeriesPage.chapter_id == target_chapter_id + ) + ) + ) or 0 + for offset, iid in enumerate(moving, start=1): + await self.session.execute( + update(SeriesPage) + .where( + and_( + SeriesPage.chapter_id == source_chapter_id, + SeriesPage.image_id == iid, + ) + ) + .values(chapter_id=target_chapter_id, page_number=max_pn + offset) + ) + await self.session.execute( + SeriesChapter.__table__.delete().where( + SeriesChapter.id == source_chapter_id + ) + ) + await self._renumber_chapters(series_tag_id) + return len(moving) + + async def set_cover(self, series_tag_id: int, image_id: int) -> None: + """Cover = the (chapter_number=1, page_number=1) image. Bring the image's + chapter to the front and the image to the front of its chapter.""" + await self._require_series(series_tag_id) + row = ( + await self.session.execute( + select(SeriesPage.chapter_id) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + SeriesPage.image_id == image_id, + ) + ) + ) + ).scalar_one_or_none() + if row is None: raise SeriesError(f"image {image_id} is not in this series") - new_order = [image_id] + [i for i in order if i != image_id] - await self.reorder(series_tag_id, new_order) + chapter_id = row + chapters = await self._chapter_order(series_tag_id) + if chapters and chapters[0] != chapter_id: + new_chapters = [chapter_id] + [c for c in chapters if c != chapter_id] + await self.reorder_chapters(series_tag_id, new_chapters) + pages = await self._chapter_page_order(chapter_id) + if pages and pages[0] != image_id: + new_pages = [image_id] + [p for p in pages if p != image_id] + await self.reorder_pages(series_tag_id, chapter_id, new_pages) + + # ---- post → series (FC-6.2) ------------------------------------------ + + async def _post_images_ordered(self, post_id: int) -> list[int]: + """A post's images in capture order (ImageRecord.primary_post_id), the + same ordering the posts feed renders thumbnails in.""" + rows = ( + await self.session.execute( + select(ImageRecord.id) + .where(ImageRecord.primary_post_id == post_id) + .order_by(ImageRecord.id.asc()) + ) + ).scalars().all() + return list(rows) + + @staticmethod + def _stated_map(image_ids: list[int], start: int | None) -> dict | None: + """Per-image stated_page = start, start+1, ... when the post stated a + starting page; None when it didn't (fall back to capture order).""" + if start is None: + return None + return {iid: start + i for i, iid in enumerate(image_ids)} + + async def promote_post_to_series(self, post_id: int) -> dict: + """Case 1: a self-contained multi-image post becomes its own series. + Creates a series tag named after the post, one chapter, the post's + images as its pages (stated pages parsed from the post when present).""" + from .tag_service import TagService + + post = await self.session.get(Post, post_id) + if post is None: + raise SeriesError(f"post {post_id} not found") + image_ids = await self._post_images_ordered(post_id) + if not image_ids: + raise SeriesError("post has no images to make a series from") + name = (post.post_title or f"Series from post {post_id}").strip()[:200] + tag = await TagService(self.session).find_or_create(name, TagKind.series) + rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}") + start, end = rng if rng else (None, None) + ch = await self.create_chapter( + tag.id, stated_page_start=start, stated_page_end=end, + ) + added = await self.add_images( + tag.id, image_ids, chapter_id=ch["id"], + stated_pages=self._stated_map(image_ids, start), + ) + return { + "series_tag_id": tag.id, "name": tag.name, + "chapter_id": ch["id"], "added": added, + } + + async def add_post_as_chapter( + self, series_tag_id: int, post_id: int + ) -> dict: + """Case 2: append a post as the next chapter of an existing series. The + chapter is titled after the post and slotted by parsed page number when + present (else appended at the end).""" + await self._require_series(series_tag_id) + post = await self.session.get(Post, post_id) + if post is None: + raise SeriesError(f"post {post_id} not found") + image_ids = await self._post_images_ordered(post_id) + if not image_ids: + raise SeriesError("post has no images to add") + rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}") + start, end = rng if rng else (None, None) + ch = await self.create_chapter( + series_tag_id, title=post.post_title or None, + stated_page_start=start, stated_page_end=end, + ) + added = await self.add_images( + series_tag_id, image_ids, chapter_id=ch["id"], + stated_pages=self._stated_map(image_ids, start), + ) + if start is not None: + await self._place_chapter_by_stated(series_tag_id, ch["id"], start) + return { + "series_tag_id": series_tag_id, "chapter_id": ch["id"], + "added": added, + } + + async def _place_chapter_by_stated( + self, series_tag_id: int, chapter_id: int, start: int + ) -> None: + """Move `chapter_id` to sit before the first chapter whose stated start + is higher — so a post stating pages 5-8 lands ahead of the 9-12 chapter.""" + rows = ( + await self.session.execute( + select(SeriesChapter.id, SeriesChapter.stated_page_start) + .where(SeriesChapter.series_tag_id == series_tag_id) + .order_by(SeriesChapter.chapter_number.asc()) + ) + ).all() + current = [r.id for r in rows] + others = [r for r in rows if r.id != chapter_id] + insert_at = len(others) + for idx, r in enumerate(others): + if r.stated_page_start is not None and r.stated_page_start > start: + insert_at = idx + break + new_order = [r.id for r in others] + new_order.insert(insert_at, chapter_id) + if new_order != current: + await self.reorder_chapters(series_tag_id, new_order) + + # ---- browse list (FC-6.2) -------------------------------------------- + + async def list_series( + self, *, sort: str = "recent", artist_id: int | None = None + ) -> list[dict]: + """Series cards for the browse view: cover thumb, name, artist, chapter + + page counts, a gap flag, and last-updated. sort ∈ recent|name|size.""" + # Page counts + most-recent activity per series. + page_rows = ( + await self.session.execute( + select( + SeriesPage.series_tag_id, + func.count().label("pages"), + ).group_by(SeriesPage.series_tag_id) + ) + ).all() + page_count = {r.series_tag_id: r.pages for r in page_rows} + + ch_rows = ( + await self.session.execute( + select( + SeriesChapter.series_tag_id, + SeriesChapter.id, + SeriesChapter.chapter_number, + SeriesChapter.stated_page_start, + SeriesChapter.stated_page_end, + SeriesChapter.updated_at, + ) + .order_by( + SeriesChapter.series_tag_id, + SeriesChapter.chapter_number.asc(), + ) + ) + ).all() + chapters_by_series: dict[int, list] = {} + updated_by_series: dict[int, object] = {} + for r in ch_rows: + chapters_by_series.setdefault(r.series_tag_id, []).append(r) + prev = updated_by_series.get(r.series_tag_id) + if prev is None or (r.updated_at and r.updated_at > prev): + updated_by_series[r.series_tag_id] = r.updated_at + + # Cover = the (chapter_number, page_number)-minimum page per series. + cover_q = ( + select( + SeriesPage.series_tag_id, + ImageRecord.sha256, + ImageRecord.mime, + ImageRecord.thumbnail_path, + ImageRecord.artist_id, + func.row_number() + .over( + partition_by=SeriesPage.series_tag_id, + order_by=( + SeriesChapter.chapter_number.asc(), + SeriesPage.page_number.asc(), + ), + ) + .label("rn"), + ) + .join(SeriesChapter, SeriesChapter.id == SeriesPage.chapter_id) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .subquery() + ) + cover_rows = ( + await self.session.execute( + select( + cover_q.c.series_tag_id, + cover_q.c.sha256, + cover_q.c.mime, + cover_q.c.thumbnail_path, + cover_q.c.artist_id, + ).where(cover_q.c.rn == 1) + ) + ).all() + cover_by_series = {r.series_tag_id: r for r in cover_rows} + + # Artist names for the covers. + artist_ids = { + r.artist_id for r in cover_rows if r.artist_id is not None + } + artist_by_id: dict[int, object] = {} + if artist_ids: + arows = ( + await self.session.execute( + select(Artist.id, Artist.name, Artist.slug) + .where(Artist.id.in_(artist_ids)) + ) + ).all() + artist_by_id = {a.id: a for a in arows} + + tags = ( + await self.session.execute( + select(Tag.id, Tag.name).where(Tag.kind == TagKind.series) + ) + ).all() + + out: list[dict] = [] + for t in tags: + cover = cover_by_series.get(t.id) + if artist_id is not None and ( + cover is None or cover.artist_id != artist_id + ): + continue + chs = chapters_by_series.get(t.id, []) + gap = bool( + self._gaps( + [ + { + "id": c.id, + "stated_page_start": c.stated_page_start, + "stated_page_end": c.stated_page_end, + } + for c in chs + ] + ) + ) + artist = artist_by_id.get(cover.artist_id) if cover else None + out.append( + { + "id": t.id, + "name": t.name, + "cover_thumbnail_url": ( + thumbnail_url( + cover.thumbnail_path, cover.sha256, cover.mime + ) + if cover + else None + ), + "artist_name": artist.name if artist else None, + "artist_slug": artist.slug if artist else None, + "chapter_count": len(chs), + "page_count": page_count.get(t.id, 0), + "has_gap": gap, + "updated_at": ( + updated_by_series.get(t.id).isoformat() + if updated_by_series.get(t.id) + else None + ), + } + ) + + if sort == "name": + out.sort(key=lambda s: s["name"].lower()) + elif sort == "size": + out.sort(key=lambda s: s["page_count"], reverse=True) + else: # recent + out.sort(key=lambda s: s["updated_at"] or "", reverse=True) + return out diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index 255311e..c798bfb 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -515,8 +515,18 @@ class TagService: ) async def _repoint_series_pages(self, src: int, tgt: int) -> None: + from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage + # Move the chapters first so the pages' chapter_id FK stays valid: a + # chapter left pointing at src would cascade-delete (with its pages) when + # src is removed. chapter_number may now collide across the merged set — + # acceptable (it's an ordering key, not unique). + await self.session.execute( + update(SeriesChapter) + .where(SeriesChapter.series_tag_id == src) + .values(series_tag_id=tgt) + ) # image_id is UNIQUE across series_page and src != tgt, so an # image in src's series cannot already be in tgt's — no collision. await self.session.execute( diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 4942c26..8096707 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -147,3 +147,54 @@ def normalize_tags_task(self) -> dict: ) normalize_tags_task.delay() return summary + + +# Time-box one rescan chunk well under the soft limit and re-enqueue from the +# cursor — scoring every post against its artist's series is O(posts) and grows +# with the library (FC-6.3). Mirrors normalize_tags_task. +_SERIES_RESCAN_CHUNK_SECONDS = 600 + + +@celery.task( + name="backend.app.tasks.admin.rescan_series_suggestions_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict: + """Score posts against their artist's series and write pending suggestions + (FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor. + Per-task async engine (NullPool) under its own asyncio loop, like normalize.""" + import asyncio + + from ..models import ImportSettings + from ..services.series_match_service import SeriesMatchService + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + try: + async with async_factory() as session: + settings = await ImportSettings.load(session) + if not settings.series_suggest_enabled: + return {"skipped": "series suggestions disabled"} + threshold = settings.series_suggest_threshold + return await SeriesMatchService(session).rescan( + threshold=threshold, + time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS, + after_post_id=after_post_id, + ) + finally: + await async_engine.dispose() + + summary = asyncio.run(_run()) + if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id: + log.info( + "rescan_series_suggestions chunk done (%d scanned, %d suggested, " + "resume after %s) — re-enqueuing", + summary.get("scanned", 0), summary.get("suggested", 0), + summary["resume_after_id"], + ) + rescan_series_suggestions_task.delay(summary["resume_after_id"]) + return summary diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 807d0e6..7e81786 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -83,8 +83,13 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days # small buffer) so the sweep never flags in-flight work. # # Backups: images backup has time_limit=23400s (6.5h). 7h covers it -# with a 30-min buffer; db backup at 12 min hard limit fits trivially. +# with a 30-min buffer. BACKUP_STALL_THRESHOLD_MINUTES = 7 * 60 +# DB backup/restore is seconds-to-minutes (35-min hard limit). It must NOT share +# the images' 7h window — a DB backup wedged on NFS would otherwise sit "running" +# for 7 hours holding the concurrency-1 maintenance_long lane (operator-flagged +# 2026-06-07). 40 min gives a small buffer over the hard limit. +BACKUP_DB_STALL_THRESHOLD_MINUTES = 40 # Library audit: scan_library_for_rule has time_limit=7500s (2h5m). # 2h15m gives a 10-min buffer. LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135 @@ -619,16 +624,20 @@ def recover_stalled_backup_runs() -> int: """ SessionLocal = _sync_session_factory() now = datetime.now(UTC) - cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES) - msg = ( - f"stranded by recovery sweep (no terminal status after " - f"{BACKUP_STALL_THRESHOLD_MINUTES // 60}h)" - ) + db_cutoff = now - timedelta(minutes=BACKUP_DB_STALL_THRESHOLD_MINUTES) + slow_cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES) + msg = "stranded by recovery sweep (no terminal status within the stall window)" with SessionLocal() as session: result = session.execute( update(BackupRun) .where(BackupRun.status.in_(["running", "restoring"])) - .where(BackupRun.started_at < cutoff) + # db backups/restores are fast (40-min window); images run hours (7h). + .where( + or_( + and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff), + and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff), + ) + ) .values(status="error", finished_at=now, error=msg) ) session.commit() diff --git a/docker-compose.yml b/docker-compose.yml index f1088ef..ee793f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,17 +21,23 @@ services: postgres: image: pgvector/pgvector:pg16 - # Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) and parallel queries - # allocate larger shared-memory segments and fail with - # "could not resize shared memory segment ... No space left on device" - # (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB). - shm_size: 512m environment: POSTGRES_USER: ${DB_USER:-fabledcurator} POSTGRES_PASSWORD: ${DB_PASSWORD:-fabledcurator_dev} POSTGRES_DB: ${DB_NAME:-fabledcurator} volumes: - postgres_data:/var/lib/postgresql/data + # Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) + parallel queries + # allocate a POSIX dynamic-shared-memory segment in /dev/shm and fail with + # "could not resize shared memory segment ... No space left on device" + # (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB). + # NOTE: a `shm_size:` key is SILENTLY IGNORED under Docker Swarm + # (`docker stack deploy` — the prod deployment here), so size /dev/shm via + # a tmpfs mount instead — honored by both Swarm and plain Compose. + - type: tmpfs + target: /dev/shm + tmpfs: + size: 536870912 # 512 MB healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-fabledcurator}"] interval: 10s diff --git a/frontend/src/components/modal/FandomPicker.vue b/frontend/src/components/modal/FandomPicker.vue index 8f1147e..cb84aa2 100644 --- a/frontend/src/components/modal/FandomPicker.vue +++ b/frontend/src/components/modal/FandomPicker.vue @@ -19,7 +19,7 @@ :item-title="(f) => f.name" :item-value="(f) => f.id" label="Fandom" clearable density="compact" - @keydown.enter="onSearchEnter" + @keydown.enter.capture="onSearchEnter" @keydown.tab="onSearchTab" /> @@ -92,12 +92,20 @@ function onConfirm () { if (f) emit('confirm', f) } -// Enter on the search field. When the menu is open Vuetify is picking the -// highlighted item (it sets selectedId + closes the menu) — don't also accept -// on that keystroke. With the menu closed and a value chosen, Enter accepts. -function onSearchEnter () { +// Enter on the search field — bound in the CAPTURE phase so this runs BEFORE +// Vuetify's own input keydown handler (which opens the menu on Enter). When the +// menu is open, let Vuetify pick the highlighted item (it sets selectedId + +// closes the menu). When it's closed and a fandom is already chosen, accept it +// and stop the event so Vuetify never (re)opens the menu — that re-open was the +// bug: Enter popped the dropdown instead of submitting (operator-flagged +// 2026-06-07). +function onSearchEnter (e) { if (menuOpen.value) return - if (selectedId.value != null) onConfirm() + if (selectedId.value != null) { + e.preventDefault() + e.stopPropagation() + onConfirm() + } } // Tab from the search field goes to the "new fandom" text field (operator- diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index cd1c2aa..c38c6da 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -15,6 +15,7 @@ · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }} + + + Series + + + + + New series from this post + + + + Add to existing series… + + + + + + + Add to series + +

+ Appends this post as the next chapter of the chosen series. +

+ + + + + Cancel + Add + + + + + + + diff --git a/frontend/src/components/settings/TagMaintenanceCard.vue b/frontend/src/components/settings/TagMaintenanceCard.vue index b1d996b..2122c58 100644 --- a/frontend/src/components/settings/TagMaintenanceCard.vue +++ b/frontend/src/components/settings/TagMaintenanceCard.vue @@ -181,7 +181,7 @@ Standardize {{ normPreview.total_changes }} tag group(s) @@ -291,7 +291,11 @@ async function onNormCommit() { // confirm it's queued; the operator can re-run Preview later to verify. await store.normalizeTags({ dryRun: false }) normResult.value = 'queued' - normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] } + // Keep the preview showing what was QUEUED — do NOT zero it out. Overwriting + // it with a zeroed object made the card read "0 tag groups to change" the + // instant Standardize was clicked, which looked like nothing happened + // (operator-flagged 2026-06-07). The button disables on `queued`; re-running + // Preview later reflects the real remaining count as the task applies. } finally { normCommitting.value = false } diff --git a/frontend/src/router.js b/frontend/src/router.js index f3b6cbe..2073c84 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -4,6 +4,7 @@ import GalleryView from './views/GalleryView.vue' import ShowcaseView from './views/ShowcaseView.vue' import TagsView from './views/TagsView.vue' import ArtistView from './views/ArtistView.vue' +import SeriesView from './views/SeriesView.vue' import SeriesManageView from './views/SeriesManageView.vue' import SeriesReaderView from './views/SeriesReaderView.vue' import SubscriptionsView from './views/SubscriptionsView.vue' @@ -25,7 +26,9 @@ const routes = [ { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, // Artist detail — no meta.title (reached by clicking an artist, not nav). { path: '/artist/:slug', name: 'artist', component: ArtistView }, - // Series management — no meta.title (reached from a series tag card). + // Series browse — a nav entry (meta.title). Peer of Posts. + { path: '/series', name: 'series', component: SeriesView, meta: { title: 'Series' } }, + // Series management — no meta.title (reached from a series card/tag). { path: '/series/:tagId', name: 'series-manage', component: SeriesManageView }, // Series reader — immersive (no top nav, no meta.title). { path: '/series/:tagId/read', name: 'series-read', component: SeriesReaderView, meta: { immersive: true } }, @@ -54,4 +57,19 @@ const router = createRouter({ routes }) +const DEFAULT_TITLE = 'FabledCurator' + +// Keep the tab title in sync with the route on EVERY navigation. List routes set +// it from meta.title; detail routes (artist, series) have no meta.title, so they +// reset to the default here and then overwrite it with their own dynamic title +// (e.g. ArtistView sets " — FabledCurator"). Without this reset the +// previous view's dynamic title stuck around — an artist name showing on the +// Showcase tab, operator-flagged 2026-06-07. +router.afterEach((to) => { + if (typeof document === 'undefined') return + document.title = to.meta?.title + ? `${to.meta.title} — ${DEFAULT_TITLE}` + : DEFAULT_TITLE +}) + export default router diff --git a/frontend/src/stores/seriesBrowse.js b/frontend/src/stores/seriesBrowse.js new file mode 100644 index 0000000..adb796c --- /dev/null +++ b/frontend/src/stores/seriesBrowse.js @@ -0,0 +1,31 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' +import { useAsyncAction } from '../composables/useAsyncAction.js' + +// Backs the Series browse view (FC-6.2). list_series returns every series as a +// card; homelab scale is small enough to load the whole list and sort/filter +// server-side. +export const useSeriesBrowseStore = defineStore('seriesBrowse', () => { + const api = useApi() + const series = ref([]) + const sort = ref('recent') + const artistId = ref(null) + const { loading, error, run } = useAsyncAction({ errorAs: 'message' }) + + async function load() { + await run(async () => { + const params = { sort: sort.value } + if (artistId.value != null) params.artist_id = artistId.value + const body = await api.get('/api/series', { params }) + series.value = body.series || [] + }) + } + + function setSort(s) { + sort.value = s + return load() + } + + return { series, sort, artistId, loading, error, load, setSort } +}) diff --git a/frontend/src/stores/seriesManage.js b/frontend/src/stores/seriesManage.js index 3967b68..9f7e628 100644 --- a/frontend/src/stores/seriesManage.js +++ b/frontend/src/stores/seriesManage.js @@ -16,7 +16,10 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { const tagId = ref(null) const series = ref(null) - const pages = ref([]) // [{image_id, page_number, thumbnail_url}] + const chapters = ref([]) // [{id, chapter_number, title, is_placeholder, stated_page_start/end, pages:[...]}] + const gaps = ref([]) // [{after_chapter_id, start, end}] + const pageCount = ref(0) + const targetChapterId = ref(null) // which chapter the picker adds into const picker = ref([]) // gallery scroll results const pickerCursor = ref(null) const pickerSelection = ref([]) // image ids @@ -28,7 +31,14 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { try { const body = await api.get(`/api/series/${id}/pages`) series.value = body.series - pages.value = body.pages + chapters.value = body.chapters || [] + gaps.value = body.gaps || [] + pageCount.value = (body.pages || []).length + // Keep a valid add-target: the selected chapter, else the first one. + const ids = chapters.value.map(c => c.id) + if (!ids.includes(targetChapterId.value)) { + targetChapterId.value = ids.length ? ids[0] : null + } } finally { loading.value = false } @@ -38,6 +48,68 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { if (tagId.value != null) await load(tagId.value) } + function gapAfter(chapterId) { + return gaps.value.find(g => g.after_chapter_id === chapterId) || null + } + + // ---- chapters ---- + async function createChapter({ title = null, isPlaceholder = false } = {}) { + await api.post(`/api/series/${tagId.value}/chapters`, { + body: { title, is_placeholder: isPlaceholder } + }) + await refresh() + } + + async function renameChapter(chapterId, title) { + await api.patch(`/api/series/${tagId.value}/chapters/${chapterId}`, { + body: { title } + }) + await refresh() + } + + async function setChapterStated(chapterId, start, end) { + await api.patch(`/api/series/${tagId.value}/chapters/${chapterId}`, { + body: { stated_page_start: start, stated_page_end: end } + }) + await refresh() + } + + async function reorderChapters(orderedChapterIds) { + await api.post(`/api/series/${tagId.value}/chapters/reorder`, { + body: { chapter_ids: orderedChapterIds } + }) + await refresh() + } + + async function moveChapter(chapterId, dir) { + const ids = chapters.value.map(c => c.id) + const from = ids.indexOf(chapterId) + const to = from + dir + if (from === -1 || to < 0 || to >= ids.length) return + await reorderChapters(moveItem(ids, from, to)) + } + + async function deleteChapter(chapterId) { + await api.delete(`/api/series/${tagId.value}/chapters/${chapterId}`) + await refresh() + } + + async function mergeChapter(sourceId, targetId) { + await api.post(`/api/series/${tagId.value}/chapters/${sourceId}/merge`, { + body: { target_chapter_id: targetId } + }) + await refresh() + toast({ text: 'Chapters merged', type: 'success' }) + } + + async function reorderPages(chapterId, orderedImageIds) { + await api.post(`/api/series/${tagId.value}/chapters/${chapterId}/reorder`, { + body: { image_ids: orderedImageIds } + }) + await refresh() + } + + // ---- picker / pages ---- async function loadPicker(reset = false) { if (reset) { picker.value = []; pickerCursor.value = null } const params = { limit: 50 } @@ -55,12 +127,13 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { async function addSelected() { if (pickerSelection.value.length === 0) return + if (targetChapterId.value == null) await createChapter() await api.post(`/api/series/${tagId.value}/pages`, { - body: { image_ids: pickerSelection.value } + body: { image_ids: pickerSelection.value, chapter_id: targetChapterId.value } }) pickerSelection.value = [] await refresh() - toast({ text: 'Added to series', type: 'success' }) + toast({ text: 'Added to chapter', type: 'success' }) } async function remove(imageId) { @@ -70,13 +143,6 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { await refresh() } - async function reorder(orderedImageIds) { - await api.post(`/api/series/${tagId.value}/reorder`, { - body: { image_ids: orderedImageIds } - }) - await refresh() - } - async function setCover(imageId) { await api.post(`/api/series/${tagId.value}/cover`, { body: { image_id: imageId } @@ -85,8 +151,11 @@ export const useSeriesManageStore = defineStore('seriesManage', () => { } return { - tagId, series, pages, picker, pickerCursor, pickerSelection, loading, - load, refresh, loadPicker, togglePick, addSelected, remove, - reorder, setCover + tagId, series, chapters, gaps, pageCount, targetChapterId, + picker, pickerCursor, pickerSelection, loading, + load, refresh, gapAfter, + createChapter, renameChapter, setChapterStated, reorderChapters, + moveChapter, deleteChapter, mergeChapter, reorderPages, + loadPicker, togglePick, addSelected, remove, setCover } }) diff --git a/frontend/src/stores/seriesReader.js b/frontend/src/stores/seriesReader.js index beac66d..86f2e98 100644 --- a/frontend/src/stores/seriesReader.js +++ b/frontend/src/stores/seriesReader.js @@ -42,7 +42,27 @@ export const useSeriesReaderStore = defineStore('seriesReader', () => { await run(async () => { const body = await api.get(`/api/series/${tagId}/pages`) series.value = body.series - pages.value = body.pages + // page_number is now WITHIN a chapter, so it can't anchor scroll/jump + // (two chapters both have a page 1). Decorate each page with a global + // `seq` (reading-order position) for anchors, plus chapter-divider info + // so the reader can mark where each chapter begins. + const chapters = body.chapters || [] + const labelById = {} + for (const c of chapters) { + labelById[c.id] = c.title || `Chapter ${c.chapter_number}` + } + let prevChapter = null + pages.value = (body.pages || []).map((p, i) => { + const isChapterStart = + chapters.length > 1 && p.chapter_id !== prevChapter + prevChapter = p.chapter_id + return { + ...p, + seq: i + 1, + isChapterStart, + chapterLabel: labelById[p.chapter_id] || null + } + }) }) } diff --git a/frontend/src/stores/seriesSuggestions.js b/frontend/src/stores/seriesSuggestions.js new file mode 100644 index 0000000..281c666 --- /dev/null +++ b/frontend/src/stores/seriesSuggestions.js @@ -0,0 +1,75 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' +import { useAsyncAction } from '../composables/useAsyncAction.js' +import { toast } from '../utils/toast.js' + +// Backs the Series → Suggestions tab (FC-6.3). The matcher is confirm-only: +// accept turns a suggestion into a chapter; skip dismisses it. enabled + +// threshold are the operator-tunable knobs (DB-backed via /settings/import). +export const useSeriesSuggestionsStore = defineStore('seriesSuggestions', () => { + const api = useApi() + const suggestions = ref([]) + const enabled = ref(true) + const threshold = ref(0.5) + const { loading, error, run } = useAsyncAction({ errorAs: 'message' }) + + async function load() { + await run(async () => { + const body = await api.get('/api/series/suggestions') + suggestions.value = body.suggestions || [] + }) + } + + async function loadSettings() { + const s = await api.get('/api/settings/import') + enabled.value = s.series_suggest_enabled + threshold.value = s.series_suggest_threshold + } + + async function saveSettings(patch) { + await api.patch('/api/settings/import', { body: patch }) + } + + async function setEnabled(v) { + enabled.value = v + await saveSettings({ series_suggest_enabled: v }) + } + + async function setThreshold(v) { + threshold.value = v + await saveSettings({ series_suggest_threshold: v }) + } + + async function accept(id) { + try { + await api.post(`/api/series/suggestions/${id}/accept`, {}) + suggestions.value = suggestions.value.filter(s => s.id !== id) + toast({ text: 'Added to series', type: 'success' }) + } catch (e) { + toast({ text: `Accept failed: ${e.message}`, type: 'error' }) + } + } + + async function dismiss(id) { + try { + await api.post(`/api/series/suggestions/${id}/dismiss`, {}) + suggestions.value = suggestions.value.filter(s => s.id !== id) + } catch (e) { + toast({ text: `Skip failed: ${e.message}`, type: 'error' }) + } + } + + async function rescan() { + await api.post('/api/series/suggestions/rescan', {}) + toast({ + text: 'Rescan queued — runs in the background; reload to see new matches.', + type: 'info' + }) + } + + return { + suggestions, enabled, threshold, loading, error, + load, loadSettings, setEnabled, setThreshold, accept, dismiss, rescan + } +}) diff --git a/frontend/src/views/SeriesManageView.vue b/frontend/src/views/SeriesManageView.vue index d12ddcc..52feba1 100644 --- a/frontend/src/views/SeriesManageView.vue +++ b/frontend/src/views/SeriesManageView.vue @@ -2,9 +2,11 @@
{{ store.series?.name || 'Series' }} - {{ store.pages.length }} page(s) + + {{ store.chapters.length }} chapter(s) · {{ store.pageCount }} page(s) +
-
-
- {{ p.page_number }} - -
- - + +
+ + +
+ No chapters yet — add one, then add images from the right.
-
- No pages yet — add images from the right. + +
+ Add chapter + + Add placeholder +
+
{{ store.pickerSelection.length }} selected @@ -42,7 +139,7 @@ size="small" color="accent" variant="flat" :disabled="store.pickerSelection.length === 0" @click="store.addSelected()" - >Add to series + >Add to {{ targetLabel }}
+ + diff --git a/frontend/test/seriesManage.spec.js b/frontend/test/seriesManage.spec.js index a094741..dc185d5 100644 --- a/frontend/test/seriesManage.spec.js +++ b/frontend/test/seriesManage.spec.js @@ -13,6 +13,19 @@ function stubFetch(handler) { }) } +const SERIES_BODY = { + series: { id: 7, name: 'V' }, + chapters: [ + { + id: 1, chapter_number: 1, title: null, is_placeholder: false, + stated_page_start: null, stated_page_end: null, + pages: [{ image_id: 1, page_number: 1, thumbnail_url: 't' }] + } + ], + gaps: [], + pages: [{ image_id: 1, page_number: 1, thumbnail_url: 't' }] +} + describe('seriesManage', () => { beforeEach(() => setActivePinia(createPinia())) afterEach(() => vi.restoreAllMocks()) @@ -22,48 +35,61 @@ describe('seriesManage', () => { expect(moveItem([1, 2, 3], 0, 2)).toEqual([2, 3, 1]) }) - it('load fetches pages + series', async () => { + it('load fetches chapters + series and picks a default target', async () => { const s = useSeriesManageStore() - stubFetch(() => ({ - status: 200, - body: { series: { id: 7, name: 'V' }, - pages: [{ image_id: 1, page_number: 1, thumbnail_url: 't' }] } - })) + stubFetch(() => ({ status: 200, body: SERIES_BODY })) await s.load(7) expect(s.series).toEqual({ id: 7, name: 'V' }) - expect(s.pages.map(p => p.image_id)).toEqual([1]) + expect(s.chapters.map(c => c.id)).toEqual([1]) + expect(s.pageCount).toBe(1) + expect(s.targetChapterId).toBe(1) }) - it('reorder posts the full ordered id list', async () => { + it('reorderPages posts ordered ids to the chapter reorder route', async () => { const s = useSeriesManageStore() s.tagId = 7 - s.pages = [{ image_id: 1 }, { image_id: 2 }, { image_id: 3 }] const calls = [] stubFetch((url, init) => { calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) if (url.includes('/reorder')) return { status: 200, body: { ok: true } } - return { status: 200, body: { series: { id: 7, name: 'V' }, pages: [] } } + return { status: 200, body: SERIES_BODY } }) - await s.reorder([3, 1, 2]) - const r = calls.find(c => c.url.includes('/reorder')) - expect(r.url).toContain('/api/series/7/reorder') - expect(r.body).toEqual({ image_ids: [3, 1, 2] }) + await s.reorderPages(3, [30, 10, 20]) + const r = calls.find(c => c.url.includes('/chapters/3/reorder')) + expect(r.url).toContain('/api/series/7/chapters/3/reorder') + expect(r.body).toEqual({ image_ids: [30, 10, 20] }) }) - it('addSelected posts picker selection then reloads', async () => { + it('moveChapter reorders via the chapter id list', async () => { const s = useSeriesManageStore() s.tagId = 7 + s.chapters = [{ id: 1 }, { id: 2 }, { id: 3 }] + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) + if (url.includes('/chapters/reorder')) return { status: 200, body: { ok: true } } + return { status: 200, body: SERIES_BODY } + }) + await s.moveChapter(2, -1) + const r = calls.find(c => c.url.includes('/chapters/reorder')) + expect(r.body).toEqual({ chapter_ids: [2, 1, 3] }) + }) + + it('addSelected posts selection + target chapter then clears', async () => { + const s = useSeriesManageStore() + s.tagId = 7 + s.targetChapterId = 5 s.pickerSelection = [9, 10] const calls = [] stubFetch((url, init) => { calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) - if (url.includes('/pages') && init.method === 'POST') + if (url.endsWith('/pages') && init.method === 'POST') return { status: 200, body: { added_count: 2 } } - return { status: 200, body: { series: { id: 7, name: 'V' }, pages: [] } } + return { status: 200, body: SERIES_BODY } }) await s.addSelected() const add = calls.find(c => c.url.endsWith('/api/series/7/pages')) - expect(add.body).toEqual({ image_ids: [9, 10] }) + expect(add.body).toEqual({ image_ids: [9, 10], chapter_id: 5 }) expect(s.pickerSelection).toEqual([]) }) }) diff --git a/tests/test_backup_service.py b/tests/test_backup_service.py index 519b23c..46b1ffc 100644 --- a/tests/test_backup_service.py +++ b/tests/test_backup_service.py @@ -5,6 +5,7 @@ without external binaries. The real subprocess behavior is exercised implicitly via the Celery task tests in test_tasks_backup.py. """ import json +import subprocess from pathlib import Path import pytest @@ -16,9 +17,9 @@ pytestmark = pytest.mark.integration @pytest.fixture def fake_subprocess(monkeypatch): - """Replace subprocess.run with a fake that writes a sentinel to - the target path (for pg_dump's -f, for tar's -cf). Captures all - calls in a list.""" + """Fake both subprocess.run (restore path) AND subprocess.Popen (the + bounded-kill backup path) so tests run without external binaries. Each + writes the target sentinel and records the cmd in a shared list.""" calls = [] class _FakeProc: @@ -26,17 +27,33 @@ def fake_subprocess(monkeypatch): stdout = b"" stderr = b"" - def _fake_run(cmd, **kwargs): - calls.append(list(cmd)) + def _write_sentinel(cmd): if cmd[0] == "pg_dump": i = cmd.index("-f") Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n") elif cmd[0] == "tar" and "-cf" in cmd: i = cmd.index("-cf") Path(cmd[i + 1]).write_bytes(b"fake tar payload") + + def _fake_run(cmd, **kwargs): + calls.append(list(cmd)) + _write_sentinel(cmd) return _FakeProc() + class _FakePopen: + def __init__(self, cmd, **kwargs): + calls.append(list(cmd)) + self.returncode = 0 + _write_sentinel(cmd) + + def communicate(self, timeout=None): + return (b"", b"") + + def kill(self): + self.returncode = -9 + monkeypatch.setattr("subprocess.run", _fake_run) + monkeypatch.setattr("subprocess.Popen", _FakePopen) return calls @@ -198,3 +215,41 @@ def test_backups_dir_created_on_first_use(tmp_path): d = backup_service._backups_dir(tmp_path) assert d.is_dir() assert d.name == "_backups" + + +# --- bounded-kill + local-temp (FC #739) ----------------------------- + + +def test_backup_db_dumps_to_local_temp_not_nfs_backups_dir(tmp_path, fake_subprocess): + """pg_dump must target a LOCAL temp path, not the (NFS) _backups dir — + so its long phase can't hang uninterruptibly on an NFS write.""" + backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path) + cmd = fake_subprocess[0] + dump_target = cmd[cmd.index("-f") + 1] + assert "_backups" not in dump_target + # The finished file still ends up in _backups (moved there). + result = backup_service.backup_db( + db_url="postgresql://u@h/d", images_root=tmp_path, + ) + assert "_backups" in result["sql_path"] + + +def test_run_bounded_fails_fast_when_unkillable(monkeypatch): + """A child stuck in D-state (communicate keeps timing out even after kill) + must NOT block the reaper — _run_bounded kills then re-raises promptly.""" + killed = {"n": 0} + + class _Hang: + def __init__(self, cmd, **kwargs): + pass + + def communicate(self, timeout=None): + raise subprocess.TimeoutExpired(cmd="x", timeout=timeout or 0) + + def kill(self): + killed["n"] += 1 + + monkeypatch.setattr("subprocess.Popen", _Hang) + with pytest.raises(subprocess.TimeoutExpired): + backup_service._run_bounded(["pg_dump"], 1) + assert killed["n"] == 1 diff --git a/tests/test_cleanup_service.py b/tests/test_cleanup_service.py index fc5db17..ba6242f 100644 --- a/tests/test_cleanup_service.py +++ b/tests/test_cleanup_service.py @@ -9,6 +9,7 @@ import pytest from sqlalchemy import func, select from backend.app.models import Artist, ImageRecord, Tag, TagKind +from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage from backend.app.models.tag import image_tag from backend.app.services import cleanup_service @@ -376,7 +377,12 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_ {"image_record_id": img.id, "tag_id": c.id}, {"image_record_id": img.id, "tag_id": s.id}, # series membership ])) - db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1)) + ch = SeriesChapter(series_tag_id=s.id, chapter_number=1) + db_sync.add(ch) + db_sync.flush() + db_sync.add(SeriesPage( + series_tag_id=s.id, chapter_id=ch.id, image_id=img.id, page_number=1 + )) db_sync.commit() result = cleanup_service.reset_content_tagging(db_sync, dry_run=False) diff --git a/tests/test_page_number_parser.py b/tests/test_page_number_parser.py new file mode 100644 index 0000000..f016c0d --- /dev/null +++ b/tests/test_page_number_parser.py @@ -0,0 +1,39 @@ +"""Pure unit tests for the stated-page parser (FC-6.2).""" + +from backend.app.services.page_number_parser import parse_page_range + + +def test_explicit_ranges(): + assert parse_page_range("pages 9-12") == (9, 12) + assert parse_page_range("pg 9–12") == (9, 12) # en-dash + assert parse_page_range("pp. 9 - 12") == (9, 12) + assert parse_page_range("Chapter 2 (pages 1-4)") == (1, 4) + + +def test_reversed_range_is_normalized(): + assert parse_page_range("pages 12-9") == (9, 12) + + +def test_single_page(): + assert parse_page_range("Page 5") == (5, 5) + assert parse_page_range("pg 5") == (5, 5) + + +def test_fraction_forms_take_the_numerator(): + assert parse_page_range("[3/8]") == (3, 3) + assert parse_page_range("(3/8)") == (3, 3) + assert parse_page_range("page 3 of 8") == (3, 3) + assert parse_page_range("pg 3/8") == (3, 3) + + +def test_no_page_keyword_is_none(): + # Bare numbers / non-page words must not match (false positives are worse). + assert parse_page_range("My Comic Part Three") is None + assert parse_page_range("Episode 5") is None + assert parse_page_range("Released 2024") is None + assert parse_page_range("Part 3") is None + + +def test_empty_and_none(): + assert parse_page_range("") is None + assert parse_page_range(None) is None diff --git a/tests/test_series_chapters.py b/tests/test_series_chapters.py new file mode 100644 index 0000000..07aa767 --- /dev/null +++ b/tests/test_series_chapters.py @@ -0,0 +1,193 @@ +"""FC-6.1 — chapter layer over series_page.""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_chapter import SeriesChapter +from backend.app.models.series_page import SeriesPage +from backend.app.services.series_service import SeriesError, SeriesService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _img(db): + global _SEQ + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_sc_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +async def _series(db, name): + return (await TagService(db).find_or_create(name, TagKind.series)).id + + +async def _chapter_numbers(db, sid): + rows = ( + await db.execute( + select(SeriesChapter.id, SeriesChapter.chapter_number) + .where(SeriesChapter.series_tag_id == sid) + .order_by(SeriesChapter.chapter_number) + ) + ).all() + return [(r.id, r.chapter_number) for r in rows] + + +async def _page_order(db, chapter_id): + rows = ( + await db.execute( + select(SeriesPage.image_id) + .where(SeriesPage.chapter_id == chapter_id) + .order_by(SeriesPage.page_number) + ) + ).scalars().all() + return list(rows) + + +@pytest.mark.asyncio +async def test_add_images_creates_default_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C1") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2]) + chapters = await _chapter_numbers(db, sid) + assert len(chapters) == 1 + assert chapters[0][1] == 1 + assert await _page_order(db, chapters[0][0]) == [i1, i2] + + +@pytest.mark.asyncio +async def test_create_and_target_specific_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C2") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # default chapter 1 + ch2 = await svc.create_chapter(sid, title="Two") + assert ch2["chapter_number"] == 2 + await svc.add_images(sid, [i2], chapter_id=ch2["id"]) + assert await _page_order(db, ch2["id"]) == [i2] + # i1 stayed in chapter 1. + chapters = await _chapter_numbers(db, sid) + assert len(chapters) == 2 + + +@pytest.mark.asyncio +async def test_reorder_chapters(db): + svc = SeriesService(db) + sid = await _series(db, "C3") + c1 = (await svc.create_chapter(sid))["id"] + c2 = (await svc.create_chapter(sid))["id"] + c3 = (await svc.create_chapter(sid))["id"] + await svc.reorder_chapters(sid, [c3, c1, c2]) + assert await _chapter_numbers(db, sid) == [(c3, 1), (c1, 2), (c2, 3)] + with pytest.raises(SeriesError): + await svc.reorder_chapters(sid, [c1, c2]) + + +@pytest.mark.asyncio +async def test_reorder_pages_within_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C4") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2, i3]) + cid = (await _chapter_numbers(db, sid))[0][0] + await svc.reorder_pages(sid, cid, [i3, i1, i2]) + assert await _page_order(db, cid) == [i3, i1, i2] + + +@pytest.mark.asyncio +async def test_legacy_reorder_rejects_multi_chapter(db): + svc = SeriesService(db) + sid = await _series(db, "C5") + i1 = await _img(db) + await svc.add_images(sid, [i1]) + await svc.create_chapter(sid) # now 2 chapters + with pytest.raises(SeriesError): + await svc.reorder(sid, [i1]) + + +@pytest.mark.asyncio +async def test_merge_chapter_moves_pages_and_renumbers(db): + svc = SeriesService(db) + sid = await _series(db, "C6") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # chapter 1 + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2, i3], chapter_id=c2) + c1 = (await _chapter_numbers(db, sid))[0][0] + moved = await svc.merge_chapter(sid, c2, c1) + assert moved == 2 + assert await _page_order(db, c1) == [i1, i2, i3] + # source chapter gone; remaining renumbered 1..N. + assert await _chapter_numbers(db, sid) == [(c1, 1)] + + +@pytest.mark.asyncio +async def test_delete_chapter_cascades_pages_and_renumbers(db): + svc = SeriesService(db) + sid = await _series(db, "C7") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1]) + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2], chapter_id=c2) + await svc.delete_chapter(sid, c2) + assert [c for _, c in await _chapter_numbers(db, sid)] == [1] + # i2's page cascaded away with its chapter. + cnt = await db.scalar( + select(func.count()).select_from(SeriesPage) + .where(SeriesPage.image_id == i2) + ) + assert cnt == 0 + + +@pytest.mark.asyncio +async def test_set_cover_brings_chapter_and_page_to_front(db): + svc = SeriesService(db) + sid = await _series(db, "C8") + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + await svc.add_images(sid, [i1]) # chapter 1 + c2 = (await svc.create_chapter(sid))["id"] + await svc.add_images(sid, [i2, i3], chapter_id=c2) + await svc.set_cover(sid, i3) + chapters = await _chapter_numbers(db, sid) + assert chapters[0][0] == c2 # i3's chapter is now first + assert (await _page_order(db, c2))[0] == i3 # i3 first in its chapter + + +@pytest.mark.asyncio +async def test_list_pages_flags_gaps_from_stated_ranges(db): + svc = SeriesService(db) + sid = await _series(db, "C9") + await svc.create_chapter(sid, stated_page_start=1, stated_page_end=4) + await svc.create_chapter(sid, stated_page_start=9, stated_page_end=12) + out = await svc.list_pages(sid) + assert len(out["chapters"]) == 2 + assert len(out["gaps"]) == 1 + assert out["gaps"][0]["start"] == 5 + assert out["gaps"][0]["end"] == 8 + + +@pytest.mark.asyncio +async def test_add_with_stated_pages_records_them(db): + svc = SeriesService(db) + sid = await _series(db, "C10") + i1, i2 = await _img(db), await _img(db) + await svc.add_images(sid, [i1, i2], stated_pages={i1: 9, i2: 10}) + rows = dict( + ( + await db.execute( + select(SeriesPage.image_id, SeriesPage.stated_page) + .where(SeriesPage.series_tag_id == sid) + ) + ).all() + ) + assert rows[i1] == 9 + assert rows[i2] == 10 diff --git a/tests/test_series_from_post.py b/tests/test_series_from_post.py new file mode 100644 index 0000000..abd63f8 --- /dev/null +++ b/tests/test_series_from_post.py @@ -0,0 +1,115 @@ +"""FC-6.2 — promote a post to a series + add a post as a chapter + browse list.""" + +import pytest + +from backend.app.models import Artist, ImageRecord, Post, TagKind +from backend.app.services.series_service import SeriesService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _artist(db, name): + a = Artist(name=name, slug=name.lower().replace(" ", "-")) + db.add(a) + await db.flush() + return a + + +async def _post(db, artist, title, ext): + p = Post(artist_id=artist.id, external_post_id=ext, post_title=title) + db.add(p) + await db.flush() + return p + + +async def _post_images(db, post, artist, n): + global _SEQ + ids = [] + for _ in range(n): + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_ps_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="downloaded", + primary_post_id=post.id, artist_id=artist.id, + ) + db.add(rec) + await db.flush() + ids.append(rec.id) + return ids + + +@pytest.mark.asyncio +async def test_promote_post_to_series(db): + svc = SeriesService(db) + artist = await _artist(db, "Bikupan") + post = await _post(db, artist, "My Comic pages 1-3", "pp1") + imgs = await _post_images(db, post, artist, 3) + + out = await svc.promote_post_to_series(post.id) + assert out["added"] == 3 + assert out["name"] == "My Comic pages 1-3" + + data = await svc.list_pages(out["series_tag_id"]) + assert len(data["chapters"]) == 1 + ch = data["chapters"][0] + assert ch["stated_page_start"] == 1 + assert ch["stated_page_end"] == 3 + assert [p["image_id"] for p in ch["pages"]] == imgs # capture order + assert [p["stated_page"] for p in ch["pages"]] == [1, 2, 3] + + +@pytest.mark.asyncio +async def test_promote_requires_images(db): + svc = SeriesService(db) + artist = await _artist(db, "Empty Artist") + post = await _post(db, artist, "No images", "pp2") + from backend.app.services.series_service import SeriesError + with pytest.raises(SeriesError): + await svc.promote_post_to_series(post.id) + + +@pytest.mark.asyncio +async def test_add_post_as_chapter_slots_by_stated_page(db): + svc = SeriesService(db) + artist = await _artist(db, "Story Artist") + sid = (await TagService(db).find_or_create("Story", TagKind.series)).id + # An existing chapter stating pages 9-12. + await svc.create_chapter(sid, stated_page_start=9, stated_page_end=12) + # A post stating pages 1-4 should slot BEFORE the 9-12 chapter. + post = await _post(db, artist, "Story pages 1-4", "pp3") + await _post_images(db, post, artist, 4) + out = await svc.add_post_as_chapter(sid, post.id) + assert out["added"] == 4 + + data = await svc.list_pages(sid) + chapters = data["chapters"] + assert chapters[0]["stated_page_start"] == 1 # new one is first + assert chapters[1]["stated_page_start"] == 9 + # gap 5-8 flagged between them + assert data["gaps"][0]["start"] == 5 + assert data["gaps"][0]["end"] == 8 + + +@pytest.mark.asyncio +async def test_list_series_cards(db): + svc = SeriesService(db) + artist = await _artist(db, "Card Artist") + post = await _post(db, artist, "Card Comic pages 1-2", "pp4") + await _post_images(db, post, artist, 2) + out = await svc.promote_post_to_series(post.id) + + rows = await svc.list_series() + card = next(r for r in rows if r["id"] == out["series_tag_id"]) + assert card["name"] == "Card Comic pages 1-2" + assert card["chapter_count"] == 1 + assert card["page_count"] == 2 + assert card["artist_name"] == "Card Artist" + assert card["cover_thumbnail_url"] + assert card["has_gap"] is False + + # artist filter keeps it; a different artist id drops it. + assert any(r["id"] == out["series_tag_id"] for r in await svc.list_series(artist_id=artist.id)) + assert all(r["id"] != out["series_tag_id"] for r in await svc.list_series(artist_id=artist.id + 99999)) diff --git a/tests/test_series_match.py b/tests/test_series_match.py new file mode 100644 index 0000000..38be348 --- /dev/null +++ b/tests/test_series_match.py @@ -0,0 +1,48 @@ +"""Pure scoring helpers for the series continuation matcher (FC-6.3).""" + +from backend.app.services.series_match_service import ( + normalize_title, + pages_signal, + tags_signal, + title_signal, + weighted_score, +) + + +def test_normalize_title_strips_installment_markers(): + assert normalize_title("Story pages 9-12") == "story" + assert normalize_title("Bunker Diaries 04") == "bunker diaries" + assert normalize_title("My Comic [3/8]") == "my comic" + assert normalize_title("") == "" + assert normalize_title(None) == "" + + +def test_title_signal(): + # Same stem after stripping page markers → full match. + assert title_signal(["Story pages 1-4"], "Story pages 9-12") == 1.0 + # Unrelated → no overlap. + assert title_signal(["My Comic"], "Other Thing") == 0.0 + # No data → 0. + assert title_signal([], "x") == 0.0 + + +def test_pages_signal(): + assert pages_signal(8, 9) == 1.0 # clean continuation + assert pages_signal(8, 10) == 0.5 # near (diff 2) + assert pages_signal(8, 11) == 0.5 # near (diff 3) + assert pages_signal(8, 20) == 0.0 # far + assert pages_signal(8, 5) == 0.0 # before / overlap + assert pages_signal(None, 9) == 0.0 + + +def test_tags_signal(): + assert tags_signal(0) == 0.0 + assert tags_signal(3) == 1.0 + assert tags_signal(6) == 1.0 # capped + assert round(tags_signal(1), 3) == 0.333 + + +def test_weighted_score(): + assert weighted_score({"title": 1, "artist": 1, "pages": 1, "tags": 1}) == 1.0 + assert weighted_score({}) == 0.0 + assert weighted_score({"title": 1.0, "artist": 1.0}) == 0.6 diff --git a/tests/test_series_match_service.py b/tests/test_series_match_service.py new file mode 100644 index 0000000..5e4970e --- /dev/null +++ b/tests/test_series_match_service.py @@ -0,0 +1,127 @@ +"""FC-6.3 — matcher writes pending suggestions; accept/dismiss lifecycle.""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import Artist, ImageRecord, Post, TagKind +from backend.app.models.series_suggestion import SeriesSuggestion +from backend.app.services.series_match_service import SeriesMatchService +from backend.app.services.series_service import SeriesService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _artist(db, name): + a = Artist(name=name, slug=name.lower().replace(" ", "-")) + db.add(a) + await db.flush() + return a + + +async def _post_with_images(db, artist, title, ext, n): + global _SEQ + p = Post(artist_id=artist.id, external_post_id=ext, post_title=title) + db.add(p) + await db.flush() + for _ in range(n): + _SEQ += 1 + db.add(ImageRecord( + path=f"/tmp/fc_sm_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="downloaded", + primary_post_id=p.id, artist_id=artist.id, + )) + await db.flush() + return p + + +async def _series_from(db, post): + out = await SeriesService(db).promote_post_to_series(post.id) + return out["series_tag_id"] + + +@pytest.mark.asyncio +async def test_match_post_suggests_continuation(db): + artist = await _artist(db, "Story Artist") + a = await _post_with_images(db, artist, "Story pages 1-4", "a", 4) + sid = await _series_from(db, a) # series stated 1-4 + b = await _post_with_images(db, artist, "Story pages 5-8", "b", 4) + + svc = SeriesMatchService(db) + made = await svc.match_post(b.id, threshold=0.5) + assert made == 1 + + pending = await svc.list_pending() + assert len(pending) == 1 + sug = pending[0] + assert sug["series_tag_id"] == sid + assert sug["post_id"] == b.id + # title 1.0 + artist 1.0 + pages 1.0 (5 == 4+1) → 0.4+0.2+0.25 = 0.85 + assert sug["score"] >= 0.85 + assert sug["signals"]["pages"] == 1.0 + + +@pytest.mark.asyncio +async def test_match_is_idempotent_under_unique(db): + artist = await _artist(db, "Idem Artist") + a = await _post_with_images(db, artist, "Tale pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Tale pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + await svc.match_post(b.id, threshold=0.5) # again + cnt = await db.scalar( + select(func.count()).select_from(SeriesSuggestion) + .where(SeriesSuggestion.post_id == b.id) + ) + assert cnt == 1 + + +@pytest.mark.asyncio +async def test_accept_adds_chapter_and_marks_added(db): + artist = await _artist(db, "Accept Artist") + a = await _post_with_images(db, artist, "Saga pages 1-4", "a", 3) + sid = await _series_from(db, a) + b = await _post_with_images(db, artist, "Saga pages 5-8", "b", 3) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.accept(sug["id"]) + data = await SeriesService(db).list_pages(sid) + assert len(data["chapters"]) == 2 # b became a 2nd chapter + status = await db.scalar( + select(SeriesSuggestion.status).where(SeriesSuggestion.id == sug["id"]) + ) + assert status == "added" + + +@pytest.mark.asyncio +async def test_dismiss_then_rematch_skips(db): + artist = await _artist(db, "Dismiss Artist") + a = await _post_with_images(db, artist, "Epic pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Epic pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.dismiss(sug["id"]) + assert await svc.list_pending() == [] + # A re-match must respect the dismissal (decided → skipped). + made = await svc.match_post(b.id, threshold=0.5) + assert made == 0 + + +@pytest.mark.asyncio +async def test_rescan_scans_posts(db): + artist = await _artist(db, "Rescan Artist") + a = await _post_with_images(db, artist, "Quest pages 1-4", "a", 2) + await _series_from(db, a) + await _post_with_images(db, artist, "Quest pages 5-8", "b", 2) + svc = SeriesMatchService(db) + summary = await svc.rescan(threshold=0.5) + assert summary["scanned"] >= 2 + assert summary["suggested"] >= 1 + assert summary["partial"] is False diff --git a/tests/test_series_model.py b/tests/test_series_model.py index ff382c5..605bebf 100644 --- a/tests/test_series_model.py +++ b/tests/test_series_model.py @@ -3,6 +3,7 @@ from sqlalchemy import select from sqlalchemy.exc import IntegrityError from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage from backend.app.services.tag_service import TagService @@ -24,10 +25,20 @@ async def _img(db): @pytest.mark.asyncio +async def _chapter(db, series_tag_id): + ch = SeriesChapter(series_tag_id=series_tag_id, chapter_number=1) + db.add(ch) + await db.flush() + return ch.id + + async def test_series_page_roundtrip_and_unique_image(db): s = await TagService(db).find_or_create("Vol1", TagKind.series) i1 = await _img(db) - db.add(SeriesPage(series_tag_id=s.id, image_id=i1, page_number=1)) + ch = await _chapter(db, s.id) + db.add(SeriesPage( + series_tag_id=s.id, chapter_id=ch, image_id=i1, page_number=1 + )) await db.flush() got = await db.scalar( select(SeriesPage.page_number).where(SeriesPage.image_id == i1) @@ -35,6 +46,9 @@ async def test_series_page_roundtrip_and_unique_image(db): assert got == 1 # UNIQUE image_id: same image can't be a page twice (any series). s2 = await TagService(db).find_or_create("Vol2", TagKind.series) - db.add(SeriesPage(series_tag_id=s2.id, image_id=i1, page_number=1)) + ch2 = await _chapter(db, s2.id) + db.add(SeriesPage( + series_tag_id=s2.id, chapter_id=ch2, image_id=i1, page_number=1 + )) with pytest.raises(IntegrityError): await db.flush() diff --git a/tests/test_tag_merge.py b/tests/test_tag_merge.py index 50a839c..ed772c3 100644 --- a/tests/test_tag_merge.py +++ b/tests/test_tag_merge.py @@ -414,16 +414,27 @@ async def test_alias_create_does_not_clobber_existing(db): @pytest.mark.asyncio async def test_merge_repoints_series_pages(db): + from backend.app.models.series_chapter import SeriesChapter from backend.app.models.series_page import SeriesPage svc = TagService(db) a = await svc.find_or_create("SerA", TagKind.series) b = await svc.find_or_create("SerB", TagKind.series) img = await _img(db) - db.add(SeriesPage(series_tag_id=a.id, image_id=img, page_number=1)) + cha = SeriesChapter(series_tag_id=a.id, chapter_number=1) + db.add(cha) + await db.flush() + db.add(SeriesPage( + series_tag_id=a.id, chapter_id=cha.id, image_id=img, page_number=1 + )) await db.flush() await svc.merge(a.id, b.id) owner = await db.scalar( select(SeriesPage.series_tag_id).where(SeriesPage.image_id == img) ) assert owner == b.id + # The chapter moved to the survivor too, so the page's FK stayed valid. + ch_owner = await db.scalar( + select(SeriesChapter.series_tag_id).where(SeriesChapter.id == cha.id) + ) + assert ch_owner == b.id diff --git a/tests/test_tasks_backup.py b/tests/test_tasks_backup.py index 4f5ace7..4c285e3 100644 --- a/tests/test_tasks_backup.py +++ b/tests/test_tasks_backup.py @@ -34,16 +34,32 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path): stdout = b"" stderr = b"" - def _fake_run(cmd, **kwargs): + def _sentinel(cmd): if cmd[0] == "pg_dump": i = cmd.index("-f") Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n") elif cmd[0] == "tar" and "-cf" in cmd: i = cmd.index("-cf") Path(cmd[i + 1]).write_bytes(b"fake tar payload") + + def _fake_run(cmd, **kwargs): + _sentinel(cmd) return _FakeProc() + class _FakePopen: + def __init__(self, cmd, **kwargs): + self.returncode = 0 + _sentinel(cmd) + + def communicate(self, timeout=None): + return (b"", b"") + + def kill(self): + self.returncode = -9 + + # backup_db/backup_images go through _run_bounded (Popen); restore via run. monkeypatch.setattr("subprocess.run", _fake_run) + monkeypatch.setattr("subprocess.Popen", _FakePopen) def _seed_backup(db_sync, *, kind, status, started_at, tag=None, @@ -89,7 +105,8 @@ async def test_backup_db_task_records_failure_on_subprocess_error(db_sync, monke def _boom(*a, **kw): raise RuntimeError("synthetic pg_dump fail") - monkeypatch.setattr("subprocess.run", _boom) + # backup_db dumps via _run_bounded → subprocess.Popen. + monkeypatch.setattr("subprocess.Popen", _boom) with pytest.raises(RuntimeError): backup_db_task.delay().get() @@ -237,6 +254,37 @@ def test_prune_backups_never_deletes_running_or_restoring(db_sync): assert set(statuses) == {"running", "restoring"} +# --- recover_stalled_backup_runs (per-kind threshold, FC #739) ------- + + +def test_stall_sweep_flips_db_fast_but_spares_running_images(db_sync): + from backend.app.tasks.maintenance import recover_stalled_backup_runs + + now = datetime.now(UTC) + # A db backup stuck 50 min → past the 40-min db window → flipped to error. + db_id = _seed_backup( + db_sync, kind="db", status="running", + started_at=now - timedelta(minutes=50), + ) + # An images backup running 50 min is still well under the 7h window → spared. + img_id = _seed_backup( + db_sync, kind="images", status="running", + started_at=now - timedelta(minutes=50), + ) + db_sync.commit() + + recover_stalled_backup_runs.apply().get() + + db_status = db_sync.execute( + select(BackupRun.status).where(BackupRun.id == db_id) + ).scalar_one() + img_status = db_sync.execute( + select(BackupRun.status).where(BackupRun.id == img_id) + ).scalar_one() + assert db_status == "error" + assert img_status == "running" + + # --- backup_db_nightly ----------------------------------------------