From c0fd80e694b271f34582ce5dfc6b2de0fe304c11 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 18:58:18 -0400 Subject: [PATCH] =?UTF-8?q?feat(series):=20assisted-continuation=20matcher?= =?UTF-8?q?=20+=20suggestion=20queue=20=E2=80=94=20backend=20(FC-6.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirm-only "this post may continue this series" matcher. - series_suggestion table (post_id, series_tag_id, score, signals jsonb, status pending|added|dismissed, UNIQUE(post,series)); migration 0041 + two settings knobs (series_suggest_enabled, series_suggest_threshold). - series_match_service: weighted additive score (title-stem / same-artist / page-continuity / shared-distinctive-tags), no single signal gating. The title "pattern" is derived on the fly from the post titles already in a series, so it sharpens as more are confirmed (no persisted state to drift). Candidates are bounded to the post's artist. match_post upserts pending suggestions (UNIQUE + on-conflict, respecting prior added/dismissed decisions). - accept reuses add_post_as_chapter then marks 'added'; dismiss marks 'dismissed'. - rescan_series_suggestions_task: settings-gated, time-boxed + self-resuming from a post-id cursor (maintenance_long lane), like normalize_tags_task. - API: GET /series/suggestions, POST ...//accept|dismiss, POST .../rescan. - Settings: enabled + threshold exposed via /settings/import. - Tests: pure scoring helpers + matcher/accept/dismiss/rescan lifecycle + UNIQUE dedup. Frontend (Suggestions tab + settings card) lands next. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0041_series_suggestions.py | 98 ++++++ backend/app/api/settings.py | 17 + backend/app/api/tags.py | 41 +++ backend/app/models/__init__.py | 2 + backend/app/models/import_settings.py | 9 + backend/app/models/series_suggestion.py | 55 +++ backend/app/services/series_match_service.py | 337 +++++++++++++++++++ backend/app/tasks/admin.py | 51 +++ tests/test_series_match.py | 48 +++ tests/test_series_match_service.py | 127 +++++++ 10 files changed, 785 insertions(+) create mode 100644 alembic/versions/0041_series_suggestions.py create mode 100644 backend/app/models/series_suggestion.py create mode 100644 backend/app/services/series_match_service.py create mode 100644 tests/test_series_match.py create mode 100644 tests/test_series_match_service.py diff --git a/alembic/versions/0041_series_suggestions.py b/alembic/versions/0041_series_suggestions.py new file mode 100644 index 0000000..51b690a --- /dev/null +++ b/alembic/versions/0041_series_suggestions.py @@ -0,0 +1,98 @@ +"""series suggestions: assisted-continuation matcher (FC-6.3) + +Revision ID: 0041 +Revises: 0040 +Create Date: 2026-06-07 + +A confirm-only queue of "this post may continue this series" hints, plus two +import_settings knobs (enable + score threshold) for the matcher. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0041" +down_revision: Union[str, None] = "0040" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_suggestion", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "post_id", + sa.Integer, + sa.ForeignKey("post.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("score", sa.Float, nullable=False), + sa.Column("signals", sa.JSON, nullable=True), + sa.Column( + "status", sa.String(16), nullable=False, server_default="pending" + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + op.create_index( + "ix_series_suggestion_post_id", "series_suggestion", ["post_id"] + ) + op.create_index( + "ix_series_suggestion_series_tag_id", + "series_suggestion", + ["series_tag_id"], + ) + op.create_index( + "ix_series_suggestion_status", "series_suggestion", ["status"] + ) + + op.add_column( + "import_settings", + sa.Column( + "series_suggest_enabled", + sa.Boolean, + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "series_suggest_threshold", + sa.Float, + nullable=False, + server_default="0.5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "series_suggest_threshold") + op.drop_column("import_settings", "series_suggest_enabled") + op.drop_index("ix_series_suggestion_status", table_name="series_suggestion") + op.drop_index( + "ix_series_suggestion_series_tag_id", table_name="series_suggestion" + ) + op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion") + op.drop_table("series_suggestion") diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index fa97bda..7d69fbf 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -25,6 +25,8 @@ _EDITABLE_FIELDS = ( "download_schedule_default_seconds", "download_event_retention_days", "download_failure_warning_threshold", + "series_suggest_enabled", + "series_suggest_threshold", ) @@ -46,6 +48,8 @@ async def get_import_settings(): "download_schedule_default_seconds": row.download_schedule_default_seconds, "download_event_retention_days": row.download_event_retention_days, "download_failure_warning_threshold": row.download_failure_warning_threshold, + "series_suggest_enabled": row.series_suggest_enabled, + "series_suggest_threshold": row.series_suggest_threshold, }) @@ -96,6 +100,19 @@ async def update_import_settings(): if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100: return _bad_int("download_failure_warning_threshold", 1, 100) + if "series_suggest_enabled" in body and not isinstance( + body["series_suggest_enabled"], bool + ): + return jsonify( + {"error": "series_suggest_enabled must be a boolean"} + ), 400 + if "series_suggest_threshold" in body: + v = body["series_suggest_threshold"] + if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: + return jsonify( + {"error": "series_suggest_threshold must be a number in [0, 1]"} + ), 400 + async with get_session() as session: row = await ImportSettings.load(session) for field in _EDITABLE_FIELDS: diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index cfbe4b7..918fa42 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -8,6 +8,7 @@ from ..extensions import get_session from ..models import Tag, TagKind from ..models.tag_allowlist import TagAllowlist from ..services.bulk_tag_service import BulkTagService +from ..services.series_match_service import SeriesMatchService from ..services.series_service import SeriesError, SeriesService from ..services.tag_directory_service import TagDirectoryService from ..services.tag_service import ( @@ -653,3 +654,43 @@ async def series_add_post(tag_id: int): return _series_err(exc) await session.commit() return jsonify(out) + + +# ---- suggestion queue (FC-6.3) -------------------------------------------- + + +@tags_bp.route("/series/suggestions", methods=["GET"]) +async def series_suggestions_list(): + async with get_session() as session: + rows = await SeriesMatchService(session).list_pending() + return jsonify({"suggestions": rows}) + + +@tags_bp.route("/series/suggestions//accept", methods=["POST"]) +async def series_suggestion_accept(sid: int): + async with get_session() as session: + try: + out = await SeriesMatchService(session).accept(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify(out) + + +@tags_bp.route("/series/suggestions//dismiss", methods=["POST"]) +async def series_suggestion_dismiss(sid: int): + async with get_session() as session: + try: + await SeriesMatchService(session).dismiss(sid) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route("/series/suggestions/rescan", methods=["POST"]) +async def series_suggestions_rescan(): + from ..tasks.admin import rescan_series_suggestions_task + + res = rescan_series_suggestions_task.delay() + return jsonify({"task_id": res.id}) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index bfde14b..00047f9 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -20,6 +20,7 @@ from .post import Post from .post_attachment import PostAttachment from .series_chapter import SeriesChapter from .series_page import SeriesPage +from .series_suggestion import SeriesSuggestion from .source import Source from .tag import Tag, TagKind, image_tag from .tag_alias import TagAlias @@ -42,6 +43,7 @@ __all__ = [ "PostAttachment", "SeriesChapter", "SeriesPage", + "SeriesSuggestion", "ImageRecord", "ImageProvenance", "Tag", diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index e5142ca..8c59fa8 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -64,6 +64,15 @@ class ImportSettings(Base): Integer, nullable=False, default=3, ) + # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is + # the weighted-score cut-off (0..1) above which a pending suggestion is made. + series_suggest_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, + ) + series_suggest_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.5, + ) + @classmethod async def load(cls, session) -> ImportSettings: """The singleton settings row (id=1), via an async session.""" diff --git a/backend/app/models/series_suggestion.py b/backend/app/models/series_suggestion.py new file mode 100644 index 0000000..316483b --- /dev/null +++ b/backend/app/models/series_suggestion.py @@ -0,0 +1,55 @@ +"""SeriesSuggestion — a confirm-only "this post may continue this series" hint. + +The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted +signals and, above the configured threshold, records a pending suggestion. The +operator confirms (→ the post is added as a chapter) or dismisses it; FC never +files a post into a series on its own. status is a plain string (no Postgres +ENUM — see the check-existing-enums lesson): pending | added | dismissed. +""" + +from datetime import datetime + +from sqlalchemy import ( + JSON, + DateTime, + Float, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesSuggestion(Base): + __tablename__ = "series_suggestion" + __table_args__ = ( + UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + score: Mapped[float] = mapped_column(Float, nullable=False) + signals: Mapped[dict | None] = mapped_column(JSON, nullable=True) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default="pending", index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/services/series_match_service.py b/backend/app/services/series_match_service.py new file mode 100644 index 0000000..364229f --- /dev/null +++ b/backend/app/services/series_match_service.py @@ -0,0 +1,337 @@ +"""FC-6.3 — assisted continuation matcher. + +Scores a (post, candidate series) pair from several weighted signals; above the +configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the +operator accepts (post becomes a chapter) or dismisses. No single signal gates; +the score is an additive weighted sum, each signal a 0..1 strength. + +The learned "title pattern" isn't persisted — it's derived on the fly from the +post titles already in a series, so it sharpens automatically as more posts are +confirmed into the series. +""" + +import re +import time + +from sqlalchemy import and_, func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord, Post, Tag, TagKind +from ..models.series_chapter import SeriesChapter +from ..models.series_page import SeriesPage +from ..models.series_suggestion import SeriesSuggestion +from ..models.tag import image_tag +from .page_number_parser import parse_page_range +from .series_service import SeriesError, SeriesService + +# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants; +# only the on/off + threshold are operator-tunable (sensitivity is the knob +# that matters; per-signal weights are an over-tune for v1). +WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15} + +_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom) +_MAX_CANDIDATES = 50 + +# Strip installment markers so titles collapse to their stable stem. +_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I) +_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]") +_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$") + + +def normalize_title(title: str | None) -> str: + if not title: + return "" + t = _PAGE_TOKEN.sub("", title) + t = _BRACKET_NUM.sub("", t) + t = _TRAILING_NUM.sub("", t) + return " ".join(t.split()).strip().lower() + + +def _common_prefix(strings: list[str]) -> str: + strings = [s for s in strings if s] + if not strings: + return "" + pre = strings[0] + for s in strings[1:]: + i = 0 + while i < len(pre) and i < len(s) and pre[i] == s[i]: + i += 1 + pre = pre[:i] + if not pre: + break + return pre.strip() + + +def title_signal(series_titles: list[str], post_title: str | None) -> float: + """Overlap of the post title against the series' title stem (the longest + common prefix of its known titles, page/installment markers removed).""" + norm = [normalize_title(t) for t in series_titles] + norm = [t for t in norm if t] + pt = normalize_title(post_title) + if not norm or not pt: + return 0.0 + stem = _common_prefix(norm) + if len(stem) < 3: + stem = max(norm, key=len) + i = 0 + while i < len(stem) and i < len(pt) and stem[i] == pt[i]: + i += 1 + return min(1.0, i / max(len(stem), 4)) + + +def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float: + """1.0 when the post's first page continues right after the series' last + stated page; partial for a near-continuation; 0 otherwise.""" + if series_max_stated_end is None or post_start is None: + return 0.0 + diff = post_start - series_max_stated_end + if diff == 1: + return 1.0 + if 2 <= diff <= 3: + return 0.5 + return 0.0 + + +def tags_signal(shared_distinctive: int) -> float: + if shared_distinctive <= 0: + return 0.0 + return min(1.0, shared_distinctive / 3.0) + + +def weighted_score(signals: dict) -> float: + return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4) + + +class SeriesMatchService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _post_image_ids(self, post_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id) + ) + ).scalars().all() + return list(rows) + + async def _series_image_ids(self, series_tag_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesPage.image_id).where( + SeriesPage.series_tag_id == series_tag_id + ) + ) + ).scalars().all() + return set(rows) + + async def _series_post_titles(self, series_tag_id: int) -> list[str]: + rows = ( + await self.session.execute( + select(Post.post_title) + .select_from(SeriesPage) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .join(Post, Post.id == ImageRecord.primary_post_id) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + Post.post_title.isnot(None), + ) + ) + .distinct() + ) + ).scalars().all() + return [t for t in rows if t] + + async def _series_max_stated_end(self, series_tag_id: int) -> int | None: + return await self.session.scalar( + select(func.max(SeriesChapter.stated_page_end)).where( + SeriesChapter.series_tag_id == series_tag_id + ) + ) + + async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]: + ids = list(image_ids) + if not ids: + return set() + rows = ( + await self.session.execute( + select(image_tag.c.tag_id) + .join(Tag, Tag.id == image_tag.c.tag_id) + .where( + and_( + image_tag.c.image_record_id.in_(ids), + Tag.kind.in_(_DISTINCTIVE_KINDS), + ) + ) + .distinct() + ) + ).scalars().all() + return set(rows) + + async def _candidate_series(self, artist_id: int) -> list[int]: + """Series that already contain a page by this artist — cross-artist + series are rare, so same-artist is the candidate bound.""" + rows = ( + await self.session.execute( + select(SeriesPage.series_tag_id) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(ImageRecord.artist_id == artist_id) + .distinct() + ) + ).scalars().all() + return list(rows) + + async def _decided_series(self, post_id: int) -> set[int]: + rows = ( + await self.session.execute( + select(SeriesSuggestion.series_tag_id).where( + and_( + SeriesSuggestion.post_id == post_id, + SeriesSuggestion.status.in_(["added", "dismissed"]), + ) + ) + ) + ).scalars().all() + return set(rows) + + async def match_post(self, post_id: int, *, threshold: float) -> int: + """Score a post against its artist's series; upsert pending suggestions + for those at/above threshold. Returns the number written.""" + post = await self.session.get(Post, post_id) + if post is None or post.artist_id is None: + return 0 + post_images = await self._post_image_ids(post_id) + if not post_images: + return 0 + post_image_set = set(post_images) + rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}") + post_start = rng[0] if rng else None + post_dtags = await self._distinctive_tags(post_images) + + decided = await self._decided_series(post_id) + candidates = await self._candidate_series(post.artist_id) + made = 0 + for sid in candidates[:_MAX_CANDIDATES]: + if sid in decided: + continue + s_images = await self._series_image_ids(sid) + if post_image_set & s_images: + continue # the post is already (partly) in this series + signals = { + "title": title_signal( + await self._series_post_titles(sid), post.post_title + ), + "artist": 1.0, # candidates are same-artist by construction + "pages": pages_signal( + await self._series_max_stated_end(sid), post_start + ), + "tags": tags_signal( + len(post_dtags & await self._distinctive_tags(s_images)) + ), + } + score = weighted_score(signals) + if score < threshold: + continue + await self.session.execute( + pg_insert(SeriesSuggestion) + .values( + post_id=post_id, series_tag_id=sid, + score=score, signals=signals, status="pending", + ) + .on_conflict_do_update( + constraint="uq_series_suggestion_post_series", + set_={"score": score, "signals": signals, "status": "pending"}, + where=SeriesSuggestion.status == "pending", + ) + ) + made += 1 + return made + + # ---- queue ops -------------------------------------------------------- + + async def list_pending(self) -> list[dict]: + rows = ( + await self.session.execute( + select( + SeriesSuggestion.id, + SeriesSuggestion.post_id, + SeriesSuggestion.series_tag_id, + SeriesSuggestion.score, + SeriesSuggestion.signals, + Post.post_title, + Post.external_post_id, + Tag.name.label("series_name"), + ) + .join(Post, Post.id == SeriesSuggestion.post_id) + .join(Tag, Tag.id == SeriesSuggestion.series_tag_id) + .where(SeriesSuggestion.status == "pending") + .order_by(SeriesSuggestion.score.desc()) + ) + ).all() + return [ + { + "id": r.id, + "post_id": r.post_id, + "series_tag_id": r.series_tag_id, + "series_name": r.series_name, + "post_title": r.post_title or f"Post {r.external_post_id}", + "score": r.score, + "signals": r.signals or {}, + } + for r in rows + ] + + async def accept(self, suggestion_id: int) -> dict: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status != "pending": + raise SeriesError(f"suggestion {suggestion_id} is already {s.status}") + out = await SeriesService(self.session).add_post_as_chapter( + s.series_tag_id, s.post_id + ) + s.status = "added" + self.session.add(s) + return out + + async def dismiss(self, suggestion_id: int) -> None: + s = await self.session.get(SeriesSuggestion, suggestion_id) + if s is None: + raise SeriesError(f"suggestion {suggestion_id} not found") + if s.status == "pending": + s.status = "dismissed" + self.session.add(s) + + async def rescan( + self, *, threshold: float, time_budget_seconds: float | None = None, + after_post_id: int = 0, + ) -> dict: + """Score every post (id > after_post_id) against its artist's series, + time-boxed + resumable like the other long maintenance sweeps.""" + post_ids = ( + await self.session.execute( + select(Post.id) + .where(Post.id > after_post_id) + .order_by(Post.id.asc()) + ) + ).scalars().all() + summary = { + "scanned": 0, "suggested": 0, + "partial": False, "resume_after_id": after_post_id, + } + start = time.monotonic() + for pid in post_ids: + summary["scanned"] += 1 + summary["resume_after_id"] = pid + summary["suggested"] += await self.match_post(pid, threshold=threshold) + await self.session.commit() # commit per post so progress survives + if ( + time_budget_seconds is not None + and time.monotonic() - start >= time_budget_seconds + ): + summary["partial"] = True + break + else: + summary["partial"] = False + return summary diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 4942c26..8096707 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -147,3 +147,54 @@ def normalize_tags_task(self) -> dict: ) normalize_tags_task.delay() return summary + + +# Time-box one rescan chunk well under the soft limit and re-enqueue from the +# cursor — scoring every post against its artist's series is O(posts) and grows +# with the library (FC-6.3). Mirrors normalize_tags_task. +_SERIES_RESCAN_CHUNK_SECONDS = 600 + + +@celery.task( + name="backend.app.tasks.admin.rescan_series_suggestions_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict: + """Score posts against their artist's series and write pending suggestions + (FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor. + Per-task async engine (NullPool) under its own asyncio loop, like normalize.""" + import asyncio + + from ..models import ImportSettings + from ..services.series_match_service import SeriesMatchService + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + try: + async with async_factory() as session: + settings = await ImportSettings.load(session) + if not settings.series_suggest_enabled: + return {"skipped": "series suggestions disabled"} + threshold = settings.series_suggest_threshold + return await SeriesMatchService(session).rescan( + threshold=threshold, + time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS, + after_post_id=after_post_id, + ) + finally: + await async_engine.dispose() + + summary = asyncio.run(_run()) + if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id: + log.info( + "rescan_series_suggestions chunk done (%d scanned, %d suggested, " + "resume after %s) — re-enqueuing", + summary.get("scanned", 0), summary.get("suggested", 0), + summary["resume_after_id"], + ) + rescan_series_suggestions_task.delay(summary["resume_after_id"]) + return summary diff --git a/tests/test_series_match.py b/tests/test_series_match.py new file mode 100644 index 0000000..38be348 --- /dev/null +++ b/tests/test_series_match.py @@ -0,0 +1,48 @@ +"""Pure scoring helpers for the series continuation matcher (FC-6.3).""" + +from backend.app.services.series_match_service import ( + normalize_title, + pages_signal, + tags_signal, + title_signal, + weighted_score, +) + + +def test_normalize_title_strips_installment_markers(): + assert normalize_title("Story pages 9-12") == "story" + assert normalize_title("Bunker Diaries 04") == "bunker diaries" + assert normalize_title("My Comic [3/8]") == "my comic" + assert normalize_title("") == "" + assert normalize_title(None) == "" + + +def test_title_signal(): + # Same stem after stripping page markers → full match. + assert title_signal(["Story pages 1-4"], "Story pages 9-12") == 1.0 + # Unrelated → no overlap. + assert title_signal(["My Comic"], "Other Thing") == 0.0 + # No data → 0. + assert title_signal([], "x") == 0.0 + + +def test_pages_signal(): + assert pages_signal(8, 9) == 1.0 # clean continuation + assert pages_signal(8, 10) == 0.5 # near (diff 2) + assert pages_signal(8, 11) == 0.5 # near (diff 3) + assert pages_signal(8, 20) == 0.0 # far + assert pages_signal(8, 5) == 0.0 # before / overlap + assert pages_signal(None, 9) == 0.0 + + +def test_tags_signal(): + assert tags_signal(0) == 0.0 + assert tags_signal(3) == 1.0 + assert tags_signal(6) == 1.0 # capped + assert round(tags_signal(1), 3) == 0.333 + + +def test_weighted_score(): + assert weighted_score({"title": 1, "artist": 1, "pages": 1, "tags": 1}) == 1.0 + assert weighted_score({}) == 0.0 + assert weighted_score({"title": 1.0, "artist": 1.0}) == 0.6 diff --git a/tests/test_series_match_service.py b/tests/test_series_match_service.py new file mode 100644 index 0000000..5e4970e --- /dev/null +++ b/tests/test_series_match_service.py @@ -0,0 +1,127 @@ +"""FC-6.3 — matcher writes pending suggestions; accept/dismiss lifecycle.""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import Artist, ImageRecord, Post, TagKind +from backend.app.models.series_suggestion import SeriesSuggestion +from backend.app.services.series_match_service import SeriesMatchService +from backend.app.services.series_service import SeriesService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _artist(db, name): + a = Artist(name=name, slug=name.lower().replace(" ", "-")) + db.add(a) + await db.flush() + return a + + +async def _post_with_images(db, artist, title, ext, n): + global _SEQ + p = Post(artist_id=artist.id, external_post_id=ext, post_title=title) + db.add(p) + await db.flush() + for _ in range(n): + _SEQ += 1 + db.add(ImageRecord( + path=f"/tmp/fc_sm_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="downloaded", + primary_post_id=p.id, artist_id=artist.id, + )) + await db.flush() + return p + + +async def _series_from(db, post): + out = await SeriesService(db).promote_post_to_series(post.id) + return out["series_tag_id"] + + +@pytest.mark.asyncio +async def test_match_post_suggests_continuation(db): + artist = await _artist(db, "Story Artist") + a = await _post_with_images(db, artist, "Story pages 1-4", "a", 4) + sid = await _series_from(db, a) # series stated 1-4 + b = await _post_with_images(db, artist, "Story pages 5-8", "b", 4) + + svc = SeriesMatchService(db) + made = await svc.match_post(b.id, threshold=0.5) + assert made == 1 + + pending = await svc.list_pending() + assert len(pending) == 1 + sug = pending[0] + assert sug["series_tag_id"] == sid + assert sug["post_id"] == b.id + # title 1.0 + artist 1.0 + pages 1.0 (5 == 4+1) → 0.4+0.2+0.25 = 0.85 + assert sug["score"] >= 0.85 + assert sug["signals"]["pages"] == 1.0 + + +@pytest.mark.asyncio +async def test_match_is_idempotent_under_unique(db): + artist = await _artist(db, "Idem Artist") + a = await _post_with_images(db, artist, "Tale pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Tale pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + await svc.match_post(b.id, threshold=0.5) # again + cnt = await db.scalar( + select(func.count()).select_from(SeriesSuggestion) + .where(SeriesSuggestion.post_id == b.id) + ) + assert cnt == 1 + + +@pytest.mark.asyncio +async def test_accept_adds_chapter_and_marks_added(db): + artist = await _artist(db, "Accept Artist") + a = await _post_with_images(db, artist, "Saga pages 1-4", "a", 3) + sid = await _series_from(db, a) + b = await _post_with_images(db, artist, "Saga pages 5-8", "b", 3) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.accept(sug["id"]) + data = await SeriesService(db).list_pages(sid) + assert len(data["chapters"]) == 2 # b became a 2nd chapter + status = await db.scalar( + select(SeriesSuggestion.status).where(SeriesSuggestion.id == sug["id"]) + ) + assert status == "added" + + +@pytest.mark.asyncio +async def test_dismiss_then_rematch_skips(db): + artist = await _artist(db, "Dismiss Artist") + a = await _post_with_images(db, artist, "Epic pages 1-4", "a", 2) + await _series_from(db, a) + b = await _post_with_images(db, artist, "Epic pages 5-8", "b", 2) + svc = SeriesMatchService(db) + await svc.match_post(b.id, threshold=0.5) + sug = (await svc.list_pending())[0] + + await svc.dismiss(sug["id"]) + assert await svc.list_pending() == [] + # A re-match must respect the dismissal (decided → skipped). + made = await svc.match_post(b.id, threshold=0.5) + assert made == 0 + + +@pytest.mark.asyncio +async def test_rescan_scans_posts(db): + artist = await _artist(db, "Rescan Artist") + a = await _post_with_images(db, artist, "Quest pages 1-4", "a", 2) + await _series_from(db, a) + await _post_with_images(db, artist, "Quest pages 5-8", "b", 2) + svc = SeriesMatchService(db) + summary = await svc.rescan(threshold=0.5) + assert summary["scanned"] >= 2 + assert summary["suggested"] >= 1 + assert summary["partial"] is False