feat(series): assisted-continuation matcher + suggestion queue — backend (FC-6.3)
Confirm-only "this post may continue this series" matcher. - series_suggestion table (post_id, series_tag_id, score, signals jsonb, status pending|added|dismissed, UNIQUE(post,series)); migration 0041 + two settings knobs (series_suggest_enabled, series_suggest_threshold). - series_match_service: weighted additive score (title-stem / same-artist / page-continuity / shared-distinctive-tags), no single signal gating. The title "pattern" is derived on the fly from the post titles already in a series, so it sharpens as more are confirmed (no persisted state to drift). Candidates are bounded to the post's artist. match_post upserts pending suggestions (UNIQUE + on-conflict, respecting prior added/dismissed decisions). - accept reuses add_post_as_chapter then marks 'added'; dismiss marks 'dismissed'. - rescan_series_suggestions_task: settings-gated, time-boxed + self-resuming from a post-id cursor (maintenance_long lane), like normalize_tags_task. - API: GET /series/suggestions, POST .../<id>/accept|dismiss, POST .../rescan. - Settings: enabled + threshold exposed via /settings/import. - Tests: pure scoring helpers + matcher/accept/dismiss/rescan lifecycle + UNIQUE dedup. Frontend (Suggestions tab + settings card) lands next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
"""series suggestions: assisted-continuation matcher (FC-6.3)
|
||||
|
||||
Revision ID: 0041
|
||||
Revises: 0040
|
||||
Create Date: 2026-06-07
|
||||
|
||||
A confirm-only queue of "this post may continue this series" hints, plus two
|
||||
import_settings knobs (enable + score threshold) for the matcher.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0041"
|
||||
down_revision: Union[str, None] = "0040"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"series_suggestion",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"post_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("post.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"series_tag_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("score", sa.Float, nullable=False),
|
||||
sa.Column("signals", sa.JSON, nullable=True),
|
||||
sa.Column(
|
||||
"status", sa.String(16), nullable=False, server_default="pending"
|
||||
),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("now()"),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("now()"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"post_id", "series_tag_id", name="uq_series_suggestion_post_series"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_series_suggestion_post_id", "series_suggestion", ["post_id"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_series_suggestion_series_tag_id",
|
||||
"series_suggestion",
|
||||
["series_tag_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_series_suggestion_status", "series_suggestion", ["status"]
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"series_suggest_enabled",
|
||||
sa.Boolean,
|
||||
nullable=False,
|
||||
server_default=sa.true(),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"series_suggest_threshold",
|
||||
sa.Float,
|
||||
nullable=False,
|
||||
server_default="0.5",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "series_suggest_threshold")
|
||||
op.drop_column("import_settings", "series_suggest_enabled")
|
||||
op.drop_index("ix_series_suggestion_status", table_name="series_suggestion")
|
||||
op.drop_index(
|
||||
"ix_series_suggestion_series_tag_id", table_name="series_suggestion"
|
||||
)
|
||||
op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion")
|
||||
op.drop_table("series_suggestion")
|
||||
@@ -25,6 +25,8 @@ _EDITABLE_FIELDS = (
|
||||
"download_schedule_default_seconds",
|
||||
"download_event_retention_days",
|
||||
"download_failure_warning_threshold",
|
||||
"series_suggest_enabled",
|
||||
"series_suggest_threshold",
|
||||
)
|
||||
|
||||
|
||||
@@ -46,6 +48,8 @@ async def get_import_settings():
|
||||
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
||||
"download_event_retention_days": row.download_event_retention_days,
|
||||
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
||||
"series_suggest_enabled": row.series_suggest_enabled,
|
||||
"series_suggest_threshold": row.series_suggest_threshold,
|
||||
})
|
||||
|
||||
|
||||
@@ -96,6 +100,19 @@ async def update_import_settings():
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
|
||||
return _bad_int("download_failure_warning_threshold", 1, 100)
|
||||
|
||||
if "series_suggest_enabled" in body and not isinstance(
|
||||
body["series_suggest_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "series_suggest_enabled must be a boolean"}
|
||||
), 400
|
||||
if "series_suggest_threshold" in body:
|
||||
v = body["series_suggest_threshold"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||
), 400
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
for field in _EDITABLE_FIELDS:
|
||||
|
||||
@@ -8,6 +8,7 @@ from ..extensions import get_session
|
||||
from ..models import Tag, TagKind
|
||||
from ..models.tag_allowlist import TagAllowlist
|
||||
from ..services.bulk_tag_service import BulkTagService
|
||||
from ..services.series_match_service import SeriesMatchService
|
||||
from ..services.series_service import SeriesError, SeriesService
|
||||
from ..services.tag_directory_service import TagDirectoryService
|
||||
from ..services.tag_service import (
|
||||
@@ -653,3 +654,43 @@ async def series_add_post(tag_id: int):
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
# ---- suggestion queue (FC-6.3) --------------------------------------------
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions", methods=["GET"])
|
||||
async def series_suggestions_list():
|
||||
async with get_session() as session:
|
||||
rows = await SeriesMatchService(session).list_pending()
|
||||
return jsonify({"suggestions": rows})
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/<int:sid>/accept", methods=["POST"])
|
||||
async def series_suggestion_accept(sid: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
out = await SeriesMatchService(session).accept(sid)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/<int:sid>/dismiss", methods=["POST"])
|
||||
async def series_suggestion_dismiss(sid: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesMatchService(session).dismiss(sid)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/rescan", methods=["POST"])
|
||||
async def series_suggestions_rescan():
|
||||
from ..tasks.admin import rescan_series_suggestions_task
|
||||
|
||||
res = rescan_series_suggestions_task.delay()
|
||||
return jsonify({"task_id": res.id})
|
||||
|
||||
@@ -20,6 +20,7 @@ from .post import Post
|
||||
from .post_attachment import PostAttachment
|
||||
from .series_chapter import SeriesChapter
|
||||
from .series_page import SeriesPage
|
||||
from .series_suggestion import SeriesSuggestion
|
||||
from .source import Source
|
||||
from .tag import Tag, TagKind, image_tag
|
||||
from .tag_alias import TagAlias
|
||||
@@ -42,6 +43,7 @@ __all__ = [
|
||||
"PostAttachment",
|
||||
"SeriesChapter",
|
||||
"SeriesPage",
|
||||
"SeriesSuggestion",
|
||||
"ImageRecord",
|
||||
"ImageProvenance",
|
||||
"Tag",
|
||||
|
||||
@@ -64,6 +64,15 @@ class ImportSettings(Base):
|
||||
Integer, nullable=False, default=3,
|
||||
)
|
||||
|
||||
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
||||
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
|
||||
series_suggest_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True,
|
||||
)
|
||||
series_suggest_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.5,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def load(cls, session) -> ImportSettings:
|
||||
"""The singleton settings row (id=1), via an async session."""
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""SeriesSuggestion — a confirm-only "this post may continue this series" hint.
|
||||
|
||||
The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted
|
||||
signals and, above the configured threshold, records a pending suggestion. The
|
||||
operator confirms (→ the post is added as a chapter) or dismisses it; FC never
|
||||
files a post into a series on its own. status is a plain string (no Postgres
|
||||
ENUM — see the check-existing-enums lesson): pending | added | dismissed.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class SeriesSuggestion(Base):
|
||||
__tablename__ = "series_suggestion"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"post_id", "series_tag_id", name="uq_series_suggestion_post_series"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
post_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
series_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
score: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
signals: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, server_default="pending", index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
@@ -0,0 +1,337 @@
|
||||
"""FC-6.3 — assisted continuation matcher.
|
||||
|
||||
Scores a (post, candidate series) pair from several weighted signals; above the
|
||||
configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the
|
||||
operator accepts (post becomes a chapter) or dismisses. No single signal gates;
|
||||
the score is an additive weighted sum, each signal a 0..1 strength.
|
||||
|
||||
The learned "title pattern" isn't persisted — it's derived on the fly from the
|
||||
post titles already in a series, so it sharpens automatically as more posts are
|
||||
confirmed into the series.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
|
||||
from sqlalchemy import and_, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import ImageRecord, Post, Tag, TagKind
|
||||
from ..models.series_chapter import SeriesChapter
|
||||
from ..models.series_page import SeriesPage
|
||||
from ..models.series_suggestion import SeriesSuggestion
|
||||
from ..models.tag import image_tag
|
||||
from .page_number_parser import parse_page_range
|
||||
from .series_service import SeriesError, SeriesService
|
||||
|
||||
# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants;
|
||||
# only the on/off + threshold are operator-tunable (sensitivity is the knob
|
||||
# that matters; per-signal weights are an over-tune for v1).
|
||||
WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15}
|
||||
|
||||
_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom)
|
||||
_MAX_CANDIDATES = 50
|
||||
|
||||
# Strip installment markers so titles collapse to their stable stem.
|
||||
_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I)
|
||||
_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]")
|
||||
_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$")
|
||||
|
||||
|
||||
def normalize_title(title: str | None) -> str:
|
||||
if not title:
|
||||
return ""
|
||||
t = _PAGE_TOKEN.sub("", title)
|
||||
t = _BRACKET_NUM.sub("", t)
|
||||
t = _TRAILING_NUM.sub("", t)
|
||||
return " ".join(t.split()).strip().lower()
|
||||
|
||||
|
||||
def _common_prefix(strings: list[str]) -> str:
|
||||
strings = [s for s in strings if s]
|
||||
if not strings:
|
||||
return ""
|
||||
pre = strings[0]
|
||||
for s in strings[1:]:
|
||||
i = 0
|
||||
while i < len(pre) and i < len(s) and pre[i] == s[i]:
|
||||
i += 1
|
||||
pre = pre[:i]
|
||||
if not pre:
|
||||
break
|
||||
return pre.strip()
|
||||
|
||||
|
||||
def title_signal(series_titles: list[str], post_title: str | None) -> float:
|
||||
"""Overlap of the post title against the series' title stem (the longest
|
||||
common prefix of its known titles, page/installment markers removed)."""
|
||||
norm = [normalize_title(t) for t in series_titles]
|
||||
norm = [t for t in norm if t]
|
||||
pt = normalize_title(post_title)
|
||||
if not norm or not pt:
|
||||
return 0.0
|
||||
stem = _common_prefix(norm)
|
||||
if len(stem) < 3:
|
||||
stem = max(norm, key=len)
|
||||
i = 0
|
||||
while i < len(stem) and i < len(pt) and stem[i] == pt[i]:
|
||||
i += 1
|
||||
return min(1.0, i / max(len(stem), 4))
|
||||
|
||||
|
||||
def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float:
|
||||
"""1.0 when the post's first page continues right after the series' last
|
||||
stated page; partial for a near-continuation; 0 otherwise."""
|
||||
if series_max_stated_end is None or post_start is None:
|
||||
return 0.0
|
||||
diff = post_start - series_max_stated_end
|
||||
if diff == 1:
|
||||
return 1.0
|
||||
if 2 <= diff <= 3:
|
||||
return 0.5
|
||||
return 0.0
|
||||
|
||||
|
||||
def tags_signal(shared_distinctive: int) -> float:
|
||||
if shared_distinctive <= 0:
|
||||
return 0.0
|
||||
return min(1.0, shared_distinctive / 3.0)
|
||||
|
||||
|
||||
def weighted_score(signals: dict) -> float:
|
||||
return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4)
|
||||
|
||||
|
||||
class SeriesMatchService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def _post_image_ids(self, post_id: int) -> list[int]:
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id)
|
||||
)
|
||||
).scalars().all()
|
||||
return list(rows)
|
||||
|
||||
async def _series_image_ids(self, series_tag_id: int) -> set[int]:
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(SeriesPage.image_id).where(
|
||||
SeriesPage.series_tag_id == series_tag_id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
async def _series_post_titles(self, series_tag_id: int) -> list[str]:
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(Post.post_title)
|
||||
.select_from(SeriesPage)
|
||||
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
||||
.join(Post, Post.id == ImageRecord.primary_post_id)
|
||||
.where(
|
||||
and_(
|
||||
SeriesPage.series_tag_id == series_tag_id,
|
||||
Post.post_title.isnot(None),
|
||||
)
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
).scalars().all()
|
||||
return [t for t in rows if t]
|
||||
|
||||
async def _series_max_stated_end(self, series_tag_id: int) -> int | None:
|
||||
return await self.session.scalar(
|
||||
select(func.max(SeriesChapter.stated_page_end)).where(
|
||||
SeriesChapter.series_tag_id == series_tag_id
|
||||
)
|
||||
)
|
||||
|
||||
async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]:
|
||||
ids = list(image_ids)
|
||||
if not ids:
|
||||
return set()
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(image_tag.c.tag_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(
|
||||
and_(
|
||||
image_tag.c.image_record_id.in_(ids),
|
||||
Tag.kind.in_(_DISTINCTIVE_KINDS),
|
||||
)
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
async def _candidate_series(self, artist_id: int) -> list[int]:
|
||||
"""Series that already contain a page by this artist — cross-artist
|
||||
series are rare, so same-artist is the candidate bound."""
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(SeriesPage.series_tag_id)
|
||||
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
||||
.where(ImageRecord.artist_id == artist_id)
|
||||
.distinct()
|
||||
)
|
||||
).scalars().all()
|
||||
return list(rows)
|
||||
|
||||
async def _decided_series(self, post_id: int) -> set[int]:
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(SeriesSuggestion.series_tag_id).where(
|
||||
and_(
|
||||
SeriesSuggestion.post_id == post_id,
|
||||
SeriesSuggestion.status.in_(["added", "dismissed"]),
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
async def match_post(self, post_id: int, *, threshold: float) -> int:
|
||||
"""Score a post against its artist's series; upsert pending suggestions
|
||||
for those at/above threshold. Returns the number written."""
|
||||
post = await self.session.get(Post, post_id)
|
||||
if post is None or post.artist_id is None:
|
||||
return 0
|
||||
post_images = await self._post_image_ids(post_id)
|
||||
if not post_images:
|
||||
return 0
|
||||
post_image_set = set(post_images)
|
||||
rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}")
|
||||
post_start = rng[0] if rng else None
|
||||
post_dtags = await self._distinctive_tags(post_images)
|
||||
|
||||
decided = await self._decided_series(post_id)
|
||||
candidates = await self._candidate_series(post.artist_id)
|
||||
made = 0
|
||||
for sid in candidates[:_MAX_CANDIDATES]:
|
||||
if sid in decided:
|
||||
continue
|
||||
s_images = await self._series_image_ids(sid)
|
||||
if post_image_set & s_images:
|
||||
continue # the post is already (partly) in this series
|
||||
signals = {
|
||||
"title": title_signal(
|
||||
await self._series_post_titles(sid), post.post_title
|
||||
),
|
||||
"artist": 1.0, # candidates are same-artist by construction
|
||||
"pages": pages_signal(
|
||||
await self._series_max_stated_end(sid), post_start
|
||||
),
|
||||
"tags": tags_signal(
|
||||
len(post_dtags & await self._distinctive_tags(s_images))
|
||||
),
|
||||
}
|
||||
score = weighted_score(signals)
|
||||
if score < threshold:
|
||||
continue
|
||||
await self.session.execute(
|
||||
pg_insert(SeriesSuggestion)
|
||||
.values(
|
||||
post_id=post_id, series_tag_id=sid,
|
||||
score=score, signals=signals, status="pending",
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_series_suggestion_post_series",
|
||||
set_={"score": score, "signals": signals, "status": "pending"},
|
||||
where=SeriesSuggestion.status == "pending",
|
||||
)
|
||||
)
|
||||
made += 1
|
||||
return made
|
||||
|
||||
# ---- queue ops --------------------------------------------------------
|
||||
|
||||
async def list_pending(self) -> list[dict]:
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(
|
||||
SeriesSuggestion.id,
|
||||
SeriesSuggestion.post_id,
|
||||
SeriesSuggestion.series_tag_id,
|
||||
SeriesSuggestion.score,
|
||||
SeriesSuggestion.signals,
|
||||
Post.post_title,
|
||||
Post.external_post_id,
|
||||
Tag.name.label("series_name"),
|
||||
)
|
||||
.join(Post, Post.id == SeriesSuggestion.post_id)
|
||||
.join(Tag, Tag.id == SeriesSuggestion.series_tag_id)
|
||||
.where(SeriesSuggestion.status == "pending")
|
||||
.order_by(SeriesSuggestion.score.desc())
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"post_id": r.post_id,
|
||||
"series_tag_id": r.series_tag_id,
|
||||
"series_name": r.series_name,
|
||||
"post_title": r.post_title or f"Post {r.external_post_id}",
|
||||
"score": r.score,
|
||||
"signals": r.signals or {},
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def accept(self, suggestion_id: int) -> dict:
|
||||
s = await self.session.get(SeriesSuggestion, suggestion_id)
|
||||
if s is None:
|
||||
raise SeriesError(f"suggestion {suggestion_id} not found")
|
||||
if s.status != "pending":
|
||||
raise SeriesError(f"suggestion {suggestion_id} is already {s.status}")
|
||||
out = await SeriesService(self.session).add_post_as_chapter(
|
||||
s.series_tag_id, s.post_id
|
||||
)
|
||||
s.status = "added"
|
||||
self.session.add(s)
|
||||
return out
|
||||
|
||||
async def dismiss(self, suggestion_id: int) -> None:
|
||||
s = await self.session.get(SeriesSuggestion, suggestion_id)
|
||||
if s is None:
|
||||
raise SeriesError(f"suggestion {suggestion_id} not found")
|
||||
if s.status == "pending":
|
||||
s.status = "dismissed"
|
||||
self.session.add(s)
|
||||
|
||||
async def rescan(
|
||||
self, *, threshold: float, time_budget_seconds: float | None = None,
|
||||
after_post_id: int = 0,
|
||||
) -> dict:
|
||||
"""Score every post (id > after_post_id) against its artist's series,
|
||||
time-boxed + resumable like the other long maintenance sweeps."""
|
||||
post_ids = (
|
||||
await self.session.execute(
|
||||
select(Post.id)
|
||||
.where(Post.id > after_post_id)
|
||||
.order_by(Post.id.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
summary = {
|
||||
"scanned": 0, "suggested": 0,
|
||||
"partial": False, "resume_after_id": after_post_id,
|
||||
}
|
||||
start = time.monotonic()
|
||||
for pid in post_ids:
|
||||
summary["scanned"] += 1
|
||||
summary["resume_after_id"] = pid
|
||||
summary["suggested"] += await self.match_post(pid, threshold=threshold)
|
||||
await self.session.commit() # commit per post so progress survives
|
||||
if (
|
||||
time_budget_seconds is not None
|
||||
and time.monotonic() - start >= time_budget_seconds
|
||||
):
|
||||
summary["partial"] = True
|
||||
break
|
||||
else:
|
||||
summary["partial"] = False
|
||||
return summary
|
||||
@@ -147,3 +147,54 @@ def normalize_tags_task(self) -> dict:
|
||||
)
|
||||
normalize_tags_task.delay()
|
||||
return summary
|
||||
|
||||
|
||||
# Time-box one rescan chunk well under the soft limit and re-enqueue from the
|
||||
# cursor — scoring every post against its artist's series is O(posts) and grows
|
||||
# with the library (FC-6.3). Mirrors normalize_tags_task.
|
||||
_SERIES_RESCAN_CHUNK_SECONDS = 600
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.rescan_series_suggestions_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict:
|
||||
"""Score posts against their artist's series and write pending suggestions
|
||||
(FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor.
|
||||
Per-task async engine (NullPool) under its own asyncio loop, like normalize."""
|
||||
import asyncio
|
||||
|
||||
from ..models import ImportSettings
|
||||
from ..services.series_match_service import SeriesMatchService
|
||||
from ._async_session import async_session_factory
|
||||
|
||||
async def _run() -> dict:
|
||||
async_factory, async_engine = async_session_factory()
|
||||
try:
|
||||
async with async_factory() as session:
|
||||
settings = await ImportSettings.load(session)
|
||||
if not settings.series_suggest_enabled:
|
||||
return {"skipped": "series suggestions disabled"}
|
||||
threshold = settings.series_suggest_threshold
|
||||
return await SeriesMatchService(session).rescan(
|
||||
threshold=threshold,
|
||||
time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS,
|
||||
after_post_id=after_post_id,
|
||||
)
|
||||
finally:
|
||||
await async_engine.dispose()
|
||||
|
||||
summary = asyncio.run(_run())
|
||||
if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id:
|
||||
log.info(
|
||||
"rescan_series_suggestions chunk done (%d scanned, %d suggested, "
|
||||
"resume after %s) — re-enqueuing",
|
||||
summary.get("scanned", 0), summary.get("suggested", 0),
|
||||
summary["resume_after_id"],
|
||||
)
|
||||
rescan_series_suggestions_task.delay(summary["resume_after_id"])
|
||||
return summary
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Pure scoring helpers for the series continuation matcher (FC-6.3)."""
|
||||
|
||||
from backend.app.services.series_match_service import (
|
||||
normalize_title,
|
||||
pages_signal,
|
||||
tags_signal,
|
||||
title_signal,
|
||||
weighted_score,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_title_strips_installment_markers():
|
||||
assert normalize_title("Story pages 9-12") == "story"
|
||||
assert normalize_title("Bunker Diaries 04") == "bunker diaries"
|
||||
assert normalize_title("My Comic [3/8]") == "my comic"
|
||||
assert normalize_title("") == ""
|
||||
assert normalize_title(None) == ""
|
||||
|
||||
|
||||
def test_title_signal():
|
||||
# Same stem after stripping page markers → full match.
|
||||
assert title_signal(["Story pages 1-4"], "Story pages 9-12") == 1.0
|
||||
# Unrelated → no overlap.
|
||||
assert title_signal(["My Comic"], "Other Thing") == 0.0
|
||||
# No data → 0.
|
||||
assert title_signal([], "x") == 0.0
|
||||
|
||||
|
||||
def test_pages_signal():
|
||||
assert pages_signal(8, 9) == 1.0 # clean continuation
|
||||
assert pages_signal(8, 10) == 0.5 # near (diff 2)
|
||||
assert pages_signal(8, 11) == 0.5 # near (diff 3)
|
||||
assert pages_signal(8, 20) == 0.0 # far
|
||||
assert pages_signal(8, 5) == 0.0 # before / overlap
|
||||
assert pages_signal(None, 9) == 0.0
|
||||
|
||||
|
||||
def test_tags_signal():
|
||||
assert tags_signal(0) == 0.0
|
||||
assert tags_signal(3) == 1.0
|
||||
assert tags_signal(6) == 1.0 # capped
|
||||
assert round(tags_signal(1), 3) == 0.333
|
||||
|
||||
|
||||
def test_weighted_score():
|
||||
assert weighted_score({"title": 1, "artist": 1, "pages": 1, "tags": 1}) == 1.0
|
||||
assert weighted_score({}) == 0.0
|
||||
assert weighted_score({"title": 1.0, "artist": 1.0}) == 0.6
|
||||
@@ -0,0 +1,127 @@
|
||||
"""FC-6.3 — matcher writes pending suggestions; accept/dismiss lifecycle."""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Post, TagKind
|
||||
from backend.app.models.series_suggestion import SeriesSuggestion
|
||||
from backend.app.services.series_match_service import SeriesMatchService
|
||||
from backend.app.services.series_service import SeriesService
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
_SEQ = 0
|
||||
|
||||
|
||||
async def _artist(db, name):
|
||||
a = Artist(name=name, slug=name.lower().replace(" ", "-"))
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
return a
|
||||
|
||||
|
||||
async def _post_with_images(db, artist, title, ext, n):
|
||||
global _SEQ
|
||||
p = Post(artist_id=artist.id, external_post_id=ext, post_title=title)
|
||||
db.add(p)
|
||||
await db.flush()
|
||||
for _ in range(n):
|
||||
_SEQ += 1
|
||||
db.add(ImageRecord(
|
||||
path=f"/tmp/fc_sm_{_SEQ}.png", sha256=f"{_SEQ:064d}",
|
||||
size_bytes=1, mime="image/png", origin="downloaded",
|
||||
primary_post_id=p.id, artist_id=artist.id,
|
||||
))
|
||||
await db.flush()
|
||||
return p
|
||||
|
||||
|
||||
async def _series_from(db, post):
|
||||
out = await SeriesService(db).promote_post_to_series(post.id)
|
||||
return out["series_tag_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_match_post_suggests_continuation(db):
|
||||
artist = await _artist(db, "Story Artist")
|
||||
a = await _post_with_images(db, artist, "Story pages 1-4", "a", 4)
|
||||
sid = await _series_from(db, a) # series stated 1-4
|
||||
b = await _post_with_images(db, artist, "Story pages 5-8", "b", 4)
|
||||
|
||||
svc = SeriesMatchService(db)
|
||||
made = await svc.match_post(b.id, threshold=0.5)
|
||||
assert made == 1
|
||||
|
||||
pending = await svc.list_pending()
|
||||
assert len(pending) == 1
|
||||
sug = pending[0]
|
||||
assert sug["series_tag_id"] == sid
|
||||
assert sug["post_id"] == b.id
|
||||
# title 1.0 + artist 1.0 + pages 1.0 (5 == 4+1) → 0.4+0.2+0.25 = 0.85
|
||||
assert sug["score"] >= 0.85
|
||||
assert sug["signals"]["pages"] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_match_is_idempotent_under_unique(db):
|
||||
artist = await _artist(db, "Idem Artist")
|
||||
a = await _post_with_images(db, artist, "Tale pages 1-4", "a", 2)
|
||||
await _series_from(db, a)
|
||||
b = await _post_with_images(db, artist, "Tale pages 5-8", "b", 2)
|
||||
svc = SeriesMatchService(db)
|
||||
await svc.match_post(b.id, threshold=0.5)
|
||||
await svc.match_post(b.id, threshold=0.5) # again
|
||||
cnt = await db.scalar(
|
||||
select(func.count()).select_from(SeriesSuggestion)
|
||||
.where(SeriesSuggestion.post_id == b.id)
|
||||
)
|
||||
assert cnt == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accept_adds_chapter_and_marks_added(db):
|
||||
artist = await _artist(db, "Accept Artist")
|
||||
a = await _post_with_images(db, artist, "Saga pages 1-4", "a", 3)
|
||||
sid = await _series_from(db, a)
|
||||
b = await _post_with_images(db, artist, "Saga pages 5-8", "b", 3)
|
||||
svc = SeriesMatchService(db)
|
||||
await svc.match_post(b.id, threshold=0.5)
|
||||
sug = (await svc.list_pending())[0]
|
||||
|
||||
await svc.accept(sug["id"])
|
||||
data = await SeriesService(db).list_pages(sid)
|
||||
assert len(data["chapters"]) == 2 # b became a 2nd chapter
|
||||
status = await db.scalar(
|
||||
select(SeriesSuggestion.status).where(SeriesSuggestion.id == sug["id"])
|
||||
)
|
||||
assert status == "added"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dismiss_then_rematch_skips(db):
|
||||
artist = await _artist(db, "Dismiss Artist")
|
||||
a = await _post_with_images(db, artist, "Epic pages 1-4", "a", 2)
|
||||
await _series_from(db, a)
|
||||
b = await _post_with_images(db, artist, "Epic pages 5-8", "b", 2)
|
||||
svc = SeriesMatchService(db)
|
||||
await svc.match_post(b.id, threshold=0.5)
|
||||
sug = (await svc.list_pending())[0]
|
||||
|
||||
await svc.dismiss(sug["id"])
|
||||
assert await svc.list_pending() == []
|
||||
# A re-match must respect the dismissal (decided → skipped).
|
||||
made = await svc.match_post(b.id, threshold=0.5)
|
||||
assert made == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rescan_scans_posts(db):
|
||||
artist = await _artist(db, "Rescan Artist")
|
||||
a = await _post_with_images(db, artist, "Quest pages 1-4", "a", 2)
|
||||
await _series_from(db, a)
|
||||
await _post_with_images(db, artist, "Quest pages 5-8", "b", 2)
|
||||
svc = SeriesMatchService(db)
|
||||
summary = await svc.rescan(threshold=0.5)
|
||||
assert summary["scanned"] >= 2
|
||||
assert summary["suggested"] >= 1
|
||||
assert summary["partial"] is False
|
||||
Reference in New Issue
Block a user