feat(series): assisted-continuation matcher + suggestion queue — backend (FC-6.3)
Confirm-only "this post may continue this series" matcher. - series_suggestion table (post_id, series_tag_id, score, signals jsonb, status pending|added|dismissed, UNIQUE(post,series)); migration 0041 + two settings knobs (series_suggest_enabled, series_suggest_threshold). - series_match_service: weighted additive score (title-stem / same-artist / page-continuity / shared-distinctive-tags), no single signal gating. The title "pattern" is derived on the fly from the post titles already in a series, so it sharpens as more are confirmed (no persisted state to drift). Candidates are bounded to the post's artist. match_post upserts pending suggestions (UNIQUE + on-conflict, respecting prior added/dismissed decisions). - accept reuses add_post_as_chapter then marks 'added'; dismiss marks 'dismissed'. - rescan_series_suggestions_task: settings-gated, time-boxed + self-resuming from a post-id cursor (maintenance_long lane), like normalize_tags_task. - API: GET /series/suggestions, POST .../<id>/accept|dismiss, POST .../rescan. - Settings: enabled + threshold exposed via /settings/import. - Tests: pure scoring helpers + matcher/accept/dismiss/rescan lifecycle + UNIQUE dedup. Frontend (Suggestions tab + settings card) lands next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,98 @@
|
|||||||
|
"""series suggestions: assisted-continuation matcher (FC-6.3)
|
||||||
|
|
||||||
|
Revision ID: 0041
|
||||||
|
Revises: 0040
|
||||||
|
Create Date: 2026-06-07
|
||||||
|
|
||||||
|
A confirm-only queue of "this post may continue this series" hints, plus two
|
||||||
|
import_settings knobs (enable + score threshold) for the matcher.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0041"
|
||||||
|
down_revision: Union[str, None] = "0040"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"series_suggestion",
|
||||||
|
sa.Column("id", sa.Integer, primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"post_id",
|
||||||
|
sa.Integer,
|
||||||
|
sa.ForeignKey("post.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"series_tag_id",
|
||||||
|
sa.Integer,
|
||||||
|
sa.ForeignKey("tag.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("score", sa.Float, nullable=False),
|
||||||
|
sa.Column("signals", sa.JSON, nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(16), nullable=False, server_default="pending"
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"post_id", "series_tag_id", name="uq_series_suggestion_post_series"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_series_suggestion_post_id", "series_suggestion", ["post_id"]
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_series_suggestion_series_tag_id",
|
||||||
|
"series_suggestion",
|
||||||
|
["series_tag_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_series_suggestion_status", "series_suggestion", ["status"]
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"series_suggest_enabled",
|
||||||
|
sa.Boolean,
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.true(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"series_suggest_threshold",
|
||||||
|
sa.Float,
|
||||||
|
nullable=False,
|
||||||
|
server_default="0.5",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_settings", "series_suggest_threshold")
|
||||||
|
op.drop_column("import_settings", "series_suggest_enabled")
|
||||||
|
op.drop_index("ix_series_suggestion_status", table_name="series_suggestion")
|
||||||
|
op.drop_index(
|
||||||
|
"ix_series_suggestion_series_tag_id", table_name="series_suggestion"
|
||||||
|
)
|
||||||
|
op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion")
|
||||||
|
op.drop_table("series_suggestion")
|
||||||
@@ -25,6 +25,8 @@ _EDITABLE_FIELDS = (
|
|||||||
"download_schedule_default_seconds",
|
"download_schedule_default_seconds",
|
||||||
"download_event_retention_days",
|
"download_event_retention_days",
|
||||||
"download_failure_warning_threshold",
|
"download_failure_warning_threshold",
|
||||||
|
"series_suggest_enabled",
|
||||||
|
"series_suggest_threshold",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -46,6 +48,8 @@ async def get_import_settings():
|
|||||||
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
||||||
"download_event_retention_days": row.download_event_retention_days,
|
"download_event_retention_days": row.download_event_retention_days,
|
||||||
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
||||||
|
"series_suggest_enabled": row.series_suggest_enabled,
|
||||||
|
"series_suggest_threshold": row.series_suggest_threshold,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
@@ -96,6 +100,19 @@ async def update_import_settings():
|
|||||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
|
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
|
||||||
return _bad_int("download_failure_warning_threshold", 1, 100)
|
return _bad_int("download_failure_warning_threshold", 1, 100)
|
||||||
|
|
||||||
|
if "series_suggest_enabled" in body and not isinstance(
|
||||||
|
body["series_suggest_enabled"], bool
|
||||||
|
):
|
||||||
|
return jsonify(
|
||||||
|
{"error": "series_suggest_enabled must be a boolean"}
|
||||||
|
), 400
|
||||||
|
if "series_suggest_threshold" in body:
|
||||||
|
v = body["series_suggest_threshold"]
|
||||||
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||||
|
return jsonify(
|
||||||
|
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||||
|
), 400
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = await ImportSettings.load(session)
|
row = await ImportSettings.load(session)
|
||||||
for field in _EDITABLE_FIELDS:
|
for field in _EDITABLE_FIELDS:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from ..extensions import get_session
|
|||||||
from ..models import Tag, TagKind
|
from ..models import Tag, TagKind
|
||||||
from ..models.tag_allowlist import TagAllowlist
|
from ..models.tag_allowlist import TagAllowlist
|
||||||
from ..services.bulk_tag_service import BulkTagService
|
from ..services.bulk_tag_service import BulkTagService
|
||||||
|
from ..services.series_match_service import SeriesMatchService
|
||||||
from ..services.series_service import SeriesError, SeriesService
|
from ..services.series_service import SeriesError, SeriesService
|
||||||
from ..services.tag_directory_service import TagDirectoryService
|
from ..services.tag_directory_service import TagDirectoryService
|
||||||
from ..services.tag_service import (
|
from ..services.tag_service import (
|
||||||
@@ -653,3 +654,43 @@ async def series_add_post(tag_id: int):
|
|||||||
return _series_err(exc)
|
return _series_err(exc)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return jsonify(out)
|
return jsonify(out)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- suggestion queue (FC-6.3) --------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@tags_bp.route("/series/suggestions", methods=["GET"])
|
||||||
|
async def series_suggestions_list():
|
||||||
|
async with get_session() as session:
|
||||||
|
rows = await SeriesMatchService(session).list_pending()
|
||||||
|
return jsonify({"suggestions": rows})
|
||||||
|
|
||||||
|
|
||||||
|
@tags_bp.route("/series/suggestions/<int:sid>/accept", methods=["POST"])
|
||||||
|
async def series_suggestion_accept(sid: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
out = await SeriesMatchService(session).accept(sid)
|
||||||
|
except SeriesError as exc:
|
||||||
|
return _series_err(exc)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify(out)
|
||||||
|
|
||||||
|
|
||||||
|
@tags_bp.route("/series/suggestions/<int:sid>/dismiss", methods=["POST"])
|
||||||
|
async def series_suggestion_dismiss(sid: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
await SeriesMatchService(session).dismiss(sid)
|
||||||
|
except SeriesError as exc:
|
||||||
|
return _series_err(exc)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"ok": True})
|
||||||
|
|
||||||
|
|
||||||
|
@tags_bp.route("/series/suggestions/rescan", methods=["POST"])
|
||||||
|
async def series_suggestions_rescan():
|
||||||
|
from ..tasks.admin import rescan_series_suggestions_task
|
||||||
|
|
||||||
|
res = rescan_series_suggestions_task.delay()
|
||||||
|
return jsonify({"task_id": res.id})
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from .post import Post
|
|||||||
from .post_attachment import PostAttachment
|
from .post_attachment import PostAttachment
|
||||||
from .series_chapter import SeriesChapter
|
from .series_chapter import SeriesChapter
|
||||||
from .series_page import SeriesPage
|
from .series_page import SeriesPage
|
||||||
|
from .series_suggestion import SeriesSuggestion
|
||||||
from .source import Source
|
from .source import Source
|
||||||
from .tag import Tag, TagKind, image_tag
|
from .tag import Tag, TagKind, image_tag
|
||||||
from .tag_alias import TagAlias
|
from .tag_alias import TagAlias
|
||||||
@@ -42,6 +43,7 @@ __all__ = [
|
|||||||
"PostAttachment",
|
"PostAttachment",
|
||||||
"SeriesChapter",
|
"SeriesChapter",
|
||||||
"SeriesPage",
|
"SeriesPage",
|
||||||
|
"SeriesSuggestion",
|
||||||
"ImageRecord",
|
"ImageRecord",
|
||||||
"ImageProvenance",
|
"ImageProvenance",
|
||||||
"Tag",
|
"Tag",
|
||||||
|
|||||||
@@ -64,6 +64,15 @@ class ImportSettings(Base):
|
|||||||
Integer, nullable=False, default=3,
|
Integer, nullable=False, default=3,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
||||||
|
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
|
||||||
|
series_suggest_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=True,
|
||||||
|
)
|
||||||
|
series_suggest_threshold: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=0.5,
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def load(cls, session) -> ImportSettings:
|
async def load(cls, session) -> ImportSettings:
|
||||||
"""The singleton settings row (id=1), via an async session."""
|
"""The singleton settings row (id=1), via an async session."""
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""SeriesSuggestion — a confirm-only "this post may continue this series" hint.
|
||||||
|
|
||||||
|
The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted
|
||||||
|
signals and, above the configured threshold, records a pending suggestion. The
|
||||||
|
operator confirms (→ the post is added as a chapter) or dismisses it; FC never
|
||||||
|
files a post into a series on its own. status is a plain string (no Postgres
|
||||||
|
ENUM — see the check-existing-enums lesson): pending | added | dismissed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class SeriesSuggestion(Base):
|
||||||
|
__tablename__ = "series_suggestion"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"post_id", "series_tag_id", name="uq_series_suggestion_post_series"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
post_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
series_tag_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
score: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
signals: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, server_default="pending", index=True
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=func.now(),
|
||||||
|
onupdate=func.now(),
|
||||||
|
)
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
"""FC-6.3 — assisted continuation matcher.
|
||||||
|
|
||||||
|
Scores a (post, candidate series) pair from several weighted signals; above the
|
||||||
|
configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the
|
||||||
|
operator accepts (post becomes a chapter) or dismisses. No single signal gates;
|
||||||
|
the score is an additive weighted sum, each signal a 0..1 strength.
|
||||||
|
|
||||||
|
The learned "title pattern" isn't persisted — it's derived on the fly from the
|
||||||
|
post titles already in a series, so it sharpens automatically as more posts are
|
||||||
|
confirmed into the series.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from sqlalchemy import and_, func, select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import ImageRecord, Post, Tag, TagKind
|
||||||
|
from ..models.series_chapter import SeriesChapter
|
||||||
|
from ..models.series_page import SeriesPage
|
||||||
|
from ..models.series_suggestion import SeriesSuggestion
|
||||||
|
from ..models.tag import image_tag
|
||||||
|
from .page_number_parser import parse_page_range
|
||||||
|
from .series_service import SeriesError, SeriesService
|
||||||
|
|
||||||
|
# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants;
|
||||||
|
# only the on/off + threshold are operator-tunable (sensitivity is the knob
|
||||||
|
# that matters; per-signal weights are an over-tune for v1).
|
||||||
|
WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15}
|
||||||
|
|
||||||
|
_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom)
|
||||||
|
_MAX_CANDIDATES = 50
|
||||||
|
|
||||||
|
# Strip installment markers so titles collapse to their stable stem.
|
||||||
|
_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I)
|
||||||
|
_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]")
|
||||||
|
_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_title(title: str | None) -> str:
|
||||||
|
if not title:
|
||||||
|
return ""
|
||||||
|
t = _PAGE_TOKEN.sub("", title)
|
||||||
|
t = _BRACKET_NUM.sub("", t)
|
||||||
|
t = _TRAILING_NUM.sub("", t)
|
||||||
|
return " ".join(t.split()).strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _common_prefix(strings: list[str]) -> str:
|
||||||
|
strings = [s for s in strings if s]
|
||||||
|
if not strings:
|
||||||
|
return ""
|
||||||
|
pre = strings[0]
|
||||||
|
for s in strings[1:]:
|
||||||
|
i = 0
|
||||||
|
while i < len(pre) and i < len(s) and pre[i] == s[i]:
|
||||||
|
i += 1
|
||||||
|
pre = pre[:i]
|
||||||
|
if not pre:
|
||||||
|
break
|
||||||
|
return pre.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def title_signal(series_titles: list[str], post_title: str | None) -> float:
|
||||||
|
"""Overlap of the post title against the series' title stem (the longest
|
||||||
|
common prefix of its known titles, page/installment markers removed)."""
|
||||||
|
norm = [normalize_title(t) for t in series_titles]
|
||||||
|
norm = [t for t in norm if t]
|
||||||
|
pt = normalize_title(post_title)
|
||||||
|
if not norm or not pt:
|
||||||
|
return 0.0
|
||||||
|
stem = _common_prefix(norm)
|
||||||
|
if len(stem) < 3:
|
||||||
|
stem = max(norm, key=len)
|
||||||
|
i = 0
|
||||||
|
while i < len(stem) and i < len(pt) and stem[i] == pt[i]:
|
||||||
|
i += 1
|
||||||
|
return min(1.0, i / max(len(stem), 4))
|
||||||
|
|
||||||
|
|
||||||
|
def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float:
|
||||||
|
"""1.0 when the post's first page continues right after the series' last
|
||||||
|
stated page; partial for a near-continuation; 0 otherwise."""
|
||||||
|
if series_max_stated_end is None or post_start is None:
|
||||||
|
return 0.0
|
||||||
|
diff = post_start - series_max_stated_end
|
||||||
|
if diff == 1:
|
||||||
|
return 1.0
|
||||||
|
if 2 <= diff <= 3:
|
||||||
|
return 0.5
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def tags_signal(shared_distinctive: int) -> float:
|
||||||
|
if shared_distinctive <= 0:
|
||||||
|
return 0.0
|
||||||
|
return min(1.0, shared_distinctive / 3.0)
|
||||||
|
|
||||||
|
|
||||||
|
def weighted_score(signals: dict) -> float:
|
||||||
|
return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4)
|
||||||
|
|
||||||
|
|
||||||
|
class SeriesMatchService:
|
||||||
|
def __init__(self, session: AsyncSession):
|
||||||
|
self.session = session
|
||||||
|
|
||||||
|
async def _post_image_ids(self, post_id: int) -> list[int]:
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return list(rows)
|
||||||
|
|
||||||
|
async def _series_image_ids(self, series_tag_id: int) -> set[int]:
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(SeriesPage.image_id).where(
|
||||||
|
SeriesPage.series_tag_id == series_tag_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return set(rows)
|
||||||
|
|
||||||
|
async def _series_post_titles(self, series_tag_id: int) -> list[str]:
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(Post.post_title)
|
||||||
|
.select_from(SeriesPage)
|
||||||
|
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
||||||
|
.join(Post, Post.id == ImageRecord.primary_post_id)
|
||||||
|
.where(
|
||||||
|
and_(
|
||||||
|
SeriesPage.series_tag_id == series_tag_id,
|
||||||
|
Post.post_title.isnot(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return [t for t in rows if t]
|
||||||
|
|
||||||
|
async def _series_max_stated_end(self, series_tag_id: int) -> int | None:
|
||||||
|
return await self.session.scalar(
|
||||||
|
select(func.max(SeriesChapter.stated_page_end)).where(
|
||||||
|
SeriesChapter.series_tag_id == series_tag_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]:
|
||||||
|
ids = list(image_ids)
|
||||||
|
if not ids:
|
||||||
|
return set()
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(image_tag.c.tag_id)
|
||||||
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||||
|
.where(
|
||||||
|
and_(
|
||||||
|
image_tag.c.image_record_id.in_(ids),
|
||||||
|
Tag.kind.in_(_DISTINCTIVE_KINDS),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return set(rows)
|
||||||
|
|
||||||
|
async def _candidate_series(self, artist_id: int) -> list[int]:
|
||||||
|
"""Series that already contain a page by this artist — cross-artist
|
||||||
|
series are rare, so same-artist is the candidate bound."""
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(SeriesPage.series_tag_id)
|
||||||
|
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
||||||
|
.where(ImageRecord.artist_id == artist_id)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return list(rows)
|
||||||
|
|
||||||
|
async def _decided_series(self, post_id: int) -> set[int]:
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(SeriesSuggestion.series_tag_id).where(
|
||||||
|
and_(
|
||||||
|
SeriesSuggestion.post_id == post_id,
|
||||||
|
SeriesSuggestion.status.in_(["added", "dismissed"]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
return set(rows)
|
||||||
|
|
||||||
|
async def match_post(self, post_id: int, *, threshold: float) -> int:
|
||||||
|
"""Score a post against its artist's series; upsert pending suggestions
|
||||||
|
for those at/above threshold. Returns the number written."""
|
||||||
|
post = await self.session.get(Post, post_id)
|
||||||
|
if post is None or post.artist_id is None:
|
||||||
|
return 0
|
||||||
|
post_images = await self._post_image_ids(post_id)
|
||||||
|
if not post_images:
|
||||||
|
return 0
|
||||||
|
post_image_set = set(post_images)
|
||||||
|
rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}")
|
||||||
|
post_start = rng[0] if rng else None
|
||||||
|
post_dtags = await self._distinctive_tags(post_images)
|
||||||
|
|
||||||
|
decided = await self._decided_series(post_id)
|
||||||
|
candidates = await self._candidate_series(post.artist_id)
|
||||||
|
made = 0
|
||||||
|
for sid in candidates[:_MAX_CANDIDATES]:
|
||||||
|
if sid in decided:
|
||||||
|
continue
|
||||||
|
s_images = await self._series_image_ids(sid)
|
||||||
|
if post_image_set & s_images:
|
||||||
|
continue # the post is already (partly) in this series
|
||||||
|
signals = {
|
||||||
|
"title": title_signal(
|
||||||
|
await self._series_post_titles(sid), post.post_title
|
||||||
|
),
|
||||||
|
"artist": 1.0, # candidates are same-artist by construction
|
||||||
|
"pages": pages_signal(
|
||||||
|
await self._series_max_stated_end(sid), post_start
|
||||||
|
),
|
||||||
|
"tags": tags_signal(
|
||||||
|
len(post_dtags & await self._distinctive_tags(s_images))
|
||||||
|
),
|
||||||
|
}
|
||||||
|
score = weighted_score(signals)
|
||||||
|
if score < threshold:
|
||||||
|
continue
|
||||||
|
await self.session.execute(
|
||||||
|
pg_insert(SeriesSuggestion)
|
||||||
|
.values(
|
||||||
|
post_id=post_id, series_tag_id=sid,
|
||||||
|
score=score, signals=signals, status="pending",
|
||||||
|
)
|
||||||
|
.on_conflict_do_update(
|
||||||
|
constraint="uq_series_suggestion_post_series",
|
||||||
|
set_={"score": score, "signals": signals, "status": "pending"},
|
||||||
|
where=SeriesSuggestion.status == "pending",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
made += 1
|
||||||
|
return made
|
||||||
|
|
||||||
|
# ---- queue ops --------------------------------------------------------
|
||||||
|
|
||||||
|
async def list_pending(self) -> list[dict]:
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(
|
||||||
|
SeriesSuggestion.id,
|
||||||
|
SeriesSuggestion.post_id,
|
||||||
|
SeriesSuggestion.series_tag_id,
|
||||||
|
SeriesSuggestion.score,
|
||||||
|
SeriesSuggestion.signals,
|
||||||
|
Post.post_title,
|
||||||
|
Post.external_post_id,
|
||||||
|
Tag.name.label("series_name"),
|
||||||
|
)
|
||||||
|
.join(Post, Post.id == SeriesSuggestion.post_id)
|
||||||
|
.join(Tag, Tag.id == SeriesSuggestion.series_tag_id)
|
||||||
|
.where(SeriesSuggestion.status == "pending")
|
||||||
|
.order_by(SeriesSuggestion.score.desc())
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": r.id,
|
||||||
|
"post_id": r.post_id,
|
||||||
|
"series_tag_id": r.series_tag_id,
|
||||||
|
"series_name": r.series_name,
|
||||||
|
"post_title": r.post_title or f"Post {r.external_post_id}",
|
||||||
|
"score": r.score,
|
||||||
|
"signals": r.signals or {},
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
async def accept(self, suggestion_id: int) -> dict:
|
||||||
|
s = await self.session.get(SeriesSuggestion, suggestion_id)
|
||||||
|
if s is None:
|
||||||
|
raise SeriesError(f"suggestion {suggestion_id} not found")
|
||||||
|
if s.status != "pending":
|
||||||
|
raise SeriesError(f"suggestion {suggestion_id} is already {s.status}")
|
||||||
|
out = await SeriesService(self.session).add_post_as_chapter(
|
||||||
|
s.series_tag_id, s.post_id
|
||||||
|
)
|
||||||
|
s.status = "added"
|
||||||
|
self.session.add(s)
|
||||||
|
return out
|
||||||
|
|
||||||
|
async def dismiss(self, suggestion_id: int) -> None:
|
||||||
|
s = await self.session.get(SeriesSuggestion, suggestion_id)
|
||||||
|
if s is None:
|
||||||
|
raise SeriesError(f"suggestion {suggestion_id} not found")
|
||||||
|
if s.status == "pending":
|
||||||
|
s.status = "dismissed"
|
||||||
|
self.session.add(s)
|
||||||
|
|
||||||
|
async def rescan(
|
||||||
|
self, *, threshold: float, time_budget_seconds: float | None = None,
|
||||||
|
after_post_id: int = 0,
|
||||||
|
) -> dict:
|
||||||
|
"""Score every post (id > after_post_id) against its artist's series,
|
||||||
|
time-boxed + resumable like the other long maintenance sweeps."""
|
||||||
|
post_ids = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(Post.id)
|
||||||
|
.where(Post.id > after_post_id)
|
||||||
|
.order_by(Post.id.asc())
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
summary = {
|
||||||
|
"scanned": 0, "suggested": 0,
|
||||||
|
"partial": False, "resume_after_id": after_post_id,
|
||||||
|
}
|
||||||
|
start = time.monotonic()
|
||||||
|
for pid in post_ids:
|
||||||
|
summary["scanned"] += 1
|
||||||
|
summary["resume_after_id"] = pid
|
||||||
|
summary["suggested"] += await self.match_post(pid, threshold=threshold)
|
||||||
|
await self.session.commit() # commit per post so progress survives
|
||||||
|
if (
|
||||||
|
time_budget_seconds is not None
|
||||||
|
and time.monotonic() - start >= time_budget_seconds
|
||||||
|
):
|
||||||
|
summary["partial"] = True
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
summary["partial"] = False
|
||||||
|
return summary
|
||||||
@@ -147,3 +147,54 @@ def normalize_tags_task(self) -> dict:
|
|||||||
)
|
)
|
||||||
normalize_tags_task.delay()
|
normalize_tags_task.delay()
|
||||||
return summary
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
# Time-box one rescan chunk well under the soft limit and re-enqueue from the
|
||||||
|
# cursor — scoring every post against its artist's series is O(posts) and grows
|
||||||
|
# with the library (FC-6.3). Mirrors normalize_tags_task.
|
||||||
|
_SERIES_RESCAN_CHUNK_SECONDS = 600
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.rescan_series_suggestions_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||||
|
)
|
||||||
|
def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict:
|
||||||
|
"""Score posts against their artist's series and write pending suggestions
|
||||||
|
(FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor.
|
||||||
|
Per-task async engine (NullPool) under its own asyncio loop, like normalize."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from ..models import ImportSettings
|
||||||
|
from ..services.series_match_service import SeriesMatchService
|
||||||
|
from ._async_session import async_session_factory
|
||||||
|
|
||||||
|
async def _run() -> dict:
|
||||||
|
async_factory, async_engine = async_session_factory()
|
||||||
|
try:
|
||||||
|
async with async_factory() as session:
|
||||||
|
settings = await ImportSettings.load(session)
|
||||||
|
if not settings.series_suggest_enabled:
|
||||||
|
return {"skipped": "series suggestions disabled"}
|
||||||
|
threshold = settings.series_suggest_threshold
|
||||||
|
return await SeriesMatchService(session).rescan(
|
||||||
|
threshold=threshold,
|
||||||
|
time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS,
|
||||||
|
after_post_id=after_post_id,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await async_engine.dispose()
|
||||||
|
|
||||||
|
summary = asyncio.run(_run())
|
||||||
|
if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id:
|
||||||
|
log.info(
|
||||||
|
"rescan_series_suggestions chunk done (%d scanned, %d suggested, "
|
||||||
|
"resume after %s) — re-enqueuing",
|
||||||
|
summary.get("scanned", 0), summary.get("suggested", 0),
|
||||||
|
summary["resume_after_id"],
|
||||||
|
)
|
||||||
|
rescan_series_suggestions_task.delay(summary["resume_after_id"])
|
||||||
|
return summary
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""Pure scoring helpers for the series continuation matcher (FC-6.3)."""
|
||||||
|
|
||||||
|
from backend.app.services.series_match_service import (
|
||||||
|
normalize_title,
|
||||||
|
pages_signal,
|
||||||
|
tags_signal,
|
||||||
|
title_signal,
|
||||||
|
weighted_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_title_strips_installment_markers():
|
||||||
|
assert normalize_title("Story pages 9-12") == "story"
|
||||||
|
assert normalize_title("Bunker Diaries 04") == "bunker diaries"
|
||||||
|
assert normalize_title("My Comic [3/8]") == "my comic"
|
||||||
|
assert normalize_title("") == ""
|
||||||
|
assert normalize_title(None) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_signal():
|
||||||
|
# Same stem after stripping page markers → full match.
|
||||||
|
assert title_signal(["Story pages 1-4"], "Story pages 9-12") == 1.0
|
||||||
|
# Unrelated → no overlap.
|
||||||
|
assert title_signal(["My Comic"], "Other Thing") == 0.0
|
||||||
|
# No data → 0.
|
||||||
|
assert title_signal([], "x") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pages_signal():
|
||||||
|
assert pages_signal(8, 9) == 1.0 # clean continuation
|
||||||
|
assert pages_signal(8, 10) == 0.5 # near (diff 2)
|
||||||
|
assert pages_signal(8, 11) == 0.5 # near (diff 3)
|
||||||
|
assert pages_signal(8, 20) == 0.0 # far
|
||||||
|
assert pages_signal(8, 5) == 0.0 # before / overlap
|
||||||
|
assert pages_signal(None, 9) == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_tags_signal():
|
||||||
|
assert tags_signal(0) == 0.0
|
||||||
|
assert tags_signal(3) == 1.0
|
||||||
|
assert tags_signal(6) == 1.0 # capped
|
||||||
|
assert round(tags_signal(1), 3) == 0.333
|
||||||
|
|
||||||
|
|
||||||
|
def test_weighted_score():
|
||||||
|
assert weighted_score({"title": 1, "artist": 1, "pages": 1, "tags": 1}) == 1.0
|
||||||
|
assert weighted_score({}) == 0.0
|
||||||
|
assert weighted_score({"title": 1.0, "artist": 1.0}) == 0.6
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
"""FC-6.3 — matcher writes pending suggestions; accept/dismiss lifecycle."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.models import Artist, ImageRecord, Post, TagKind
|
||||||
|
from backend.app.models.series_suggestion import SeriesSuggestion
|
||||||
|
from backend.app.services.series_match_service import SeriesMatchService
|
||||||
|
from backend.app.services.series_service import SeriesService
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
_SEQ = 0
|
||||||
|
|
||||||
|
|
||||||
|
async def _artist(db, name):
|
||||||
|
a = Artist(name=name, slug=name.lower().replace(" ", "-"))
|
||||||
|
db.add(a)
|
||||||
|
await db.flush()
|
||||||
|
return a
|
||||||
|
|
||||||
|
|
||||||
|
async def _post_with_images(db, artist, title, ext, n):
|
||||||
|
global _SEQ
|
||||||
|
p = Post(artist_id=artist.id, external_post_id=ext, post_title=title)
|
||||||
|
db.add(p)
|
||||||
|
await db.flush()
|
||||||
|
for _ in range(n):
|
||||||
|
_SEQ += 1
|
||||||
|
db.add(ImageRecord(
|
||||||
|
path=f"/tmp/fc_sm_{_SEQ}.png", sha256=f"{_SEQ:064d}",
|
||||||
|
size_bytes=1, mime="image/png", origin="downloaded",
|
||||||
|
primary_post_id=p.id, artist_id=artist.id,
|
||||||
|
))
|
||||||
|
await db.flush()
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
async def _series_from(db, post):
|
||||||
|
out = await SeriesService(db).promote_post_to_series(post.id)
|
||||||
|
return out["series_tag_id"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_match_post_suggests_continuation(db):
|
||||||
|
artist = await _artist(db, "Story Artist")
|
||||||
|
a = await _post_with_images(db, artist, "Story pages 1-4", "a", 4)
|
||||||
|
sid = await _series_from(db, a) # series stated 1-4
|
||||||
|
b = await _post_with_images(db, artist, "Story pages 5-8", "b", 4)
|
||||||
|
|
||||||
|
svc = SeriesMatchService(db)
|
||||||
|
made = await svc.match_post(b.id, threshold=0.5)
|
||||||
|
assert made == 1
|
||||||
|
|
||||||
|
pending = await svc.list_pending()
|
||||||
|
assert len(pending) == 1
|
||||||
|
sug = pending[0]
|
||||||
|
assert sug["series_tag_id"] == sid
|
||||||
|
assert sug["post_id"] == b.id
|
||||||
|
# title 1.0 + artist 1.0 + pages 1.0 (5 == 4+1) → 0.4+0.2+0.25 = 0.85
|
||||||
|
assert sug["score"] >= 0.85
|
||||||
|
assert sug["signals"]["pages"] == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_match_is_idempotent_under_unique(db):
|
||||||
|
artist = await _artist(db, "Idem Artist")
|
||||||
|
a = await _post_with_images(db, artist, "Tale pages 1-4", "a", 2)
|
||||||
|
await _series_from(db, a)
|
||||||
|
b = await _post_with_images(db, artist, "Tale pages 5-8", "b", 2)
|
||||||
|
svc = SeriesMatchService(db)
|
||||||
|
await svc.match_post(b.id, threshold=0.5)
|
||||||
|
await svc.match_post(b.id, threshold=0.5) # again
|
||||||
|
cnt = await db.scalar(
|
||||||
|
select(func.count()).select_from(SeriesSuggestion)
|
||||||
|
.where(SeriesSuggestion.post_id == b.id)
|
||||||
|
)
|
||||||
|
assert cnt == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_accept_adds_chapter_and_marks_added(db):
|
||||||
|
artist = await _artist(db, "Accept Artist")
|
||||||
|
a = await _post_with_images(db, artist, "Saga pages 1-4", "a", 3)
|
||||||
|
sid = await _series_from(db, a)
|
||||||
|
b = await _post_with_images(db, artist, "Saga pages 5-8", "b", 3)
|
||||||
|
svc = SeriesMatchService(db)
|
||||||
|
await svc.match_post(b.id, threshold=0.5)
|
||||||
|
sug = (await svc.list_pending())[0]
|
||||||
|
|
||||||
|
await svc.accept(sug["id"])
|
||||||
|
data = await SeriesService(db).list_pages(sid)
|
||||||
|
assert len(data["chapters"]) == 2 # b became a 2nd chapter
|
||||||
|
status = await db.scalar(
|
||||||
|
select(SeriesSuggestion.status).where(SeriesSuggestion.id == sug["id"])
|
||||||
|
)
|
||||||
|
assert status == "added"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dismiss_then_rematch_skips(db):
|
||||||
|
artist = await _artist(db, "Dismiss Artist")
|
||||||
|
a = await _post_with_images(db, artist, "Epic pages 1-4", "a", 2)
|
||||||
|
await _series_from(db, a)
|
||||||
|
b = await _post_with_images(db, artist, "Epic pages 5-8", "b", 2)
|
||||||
|
svc = SeriesMatchService(db)
|
||||||
|
await svc.match_post(b.id, threshold=0.5)
|
||||||
|
sug = (await svc.list_pending())[0]
|
||||||
|
|
||||||
|
await svc.dismiss(sug["id"])
|
||||||
|
assert await svc.list_pending() == []
|
||||||
|
# A re-match must respect the dismissal (decided → skipped).
|
||||||
|
made = await svc.match_post(b.id, threshold=0.5)
|
||||||
|
assert made == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rescan_scans_posts(db):
|
||||||
|
artist = await _artist(db, "Rescan Artist")
|
||||||
|
a = await _post_with_images(db, artist, "Quest pages 1-4", "a", 2)
|
||||||
|
await _series_from(db, a)
|
||||||
|
await _post_with_images(db, artist, "Quest pages 5-8", "b", 2)
|
||||||
|
svc = SeriesMatchService(db)
|
||||||
|
summary = await svc.rescan(threshold=0.5)
|
||||||
|
assert summary["scanned"] >= 2
|
||||||
|
assert summary["suggested"] >= 1
|
||||||
|
assert summary["partial"] is False
|
||||||
Reference in New Issue
Block a user