diff --git a/.gitignore b/.gitignore index 10d4d4b..c726541 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ Thumbs.db alembic/versions/__pycache__/ *.sqlite *.sqlite-journal +.superpowers/ diff --git a/alembic/env.py b/alembic/env.py index 0530946..b657465 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -2,12 +2,21 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config, pool +from sqlalchemy import engine_from_config, pool, text from alembic import context from backend.app.config import get_config from backend.app.models import Base +# Arbitrary fixed 64-bit key for the session/transaction advisory lock that +# serializes concurrent `alembic upgrade head` runs. Every `web` replica runs +# migrations in its entrypoint, so under `docker stack deploy` two replicas can +# boot at once and race the same DDL — duplicate CREATE TABLE, then a crashed +# replica (operator-flagged 2026-06-07: 0040 raced; one backend died with +# AdminShutdown). The first replica to reach the lock migrates; the rest block, +# then find the version table already at head and apply nothing. +_MIGRATION_LOCK_KEY = 0xFCA1E35C + config = context.config if config.config_file_name is not None: @@ -44,6 +53,18 @@ def run_migrations_online() -> None: compare_type=True, ) with context.begin_transaction(): + # Serialize concurrent migrators (see _MIGRATION_LOCK_KEY). A + # transaction-scoped advisory lock: the first replica to get here + # holds it for the whole upgrade and is auto-released when this + # transaction ends. A sibling replica blocks on this line, and only + # once the leader commits does it proceed to read the version table + # — now at head — so it runs zero migrations instead of re-applying + # the same DDL. The lock is acquired BEFORE run_migrations() reads + # the current revision, which is what makes the no-op correct. + connection.execute( + text("SELECT pg_advisory_xact_lock(:k)"), + {"k": _MIGRATION_LOCK_KEY}, + ) context.run_migrations() diff --git a/alembic/versions/0042_series_chapter_stated_part.py b/alembic/versions/0042_series_chapter_stated_part.py new file mode 100644 index 0000000..f898e56 --- /dev/null +++ b/alembic/versions/0042_series_chapter_stated_part.py @@ -0,0 +1,32 @@ +"""series chapter stated_part: operator-facing Part N label (FC-6.4) + +Revision ID: 0042 +Revises: 0041 +Create Date: 2026-06-07 + +A chapter's positional chapter_number is auto-managed (rewritten 1..N on +reorder/delete), so it can't double as the installment number the operator wants +to type (e.g. a series authored from a post that is Part 2). Add a nullable +stated_part alongside it — the same split as series_page.page_number (order) vs +series_page.stated_page (printed number). Nullable; the UI falls back to +chapter_number when unset. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0042" +down_revision: Union[str, None] = "0041" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "series_chapter", sa.Column("stated_part", sa.Integer, nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("series_chapter", "stated_part") diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 918fa42..59dc286 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -528,6 +528,11 @@ async def series_chapter_update(tag_id: int, chapter_id: int): if body["title"] is not None and not isinstance(body["title"], str): return jsonify({"error": "title must be a string"}), 400 kwargs.update(set_title=True, title=body["title"]) + if "stated_part" in body: + part, perr = _opt_int(body, "stated_part") + if perr: + return perr + kwargs.update(set_part=True, stated_part=part) if "stated_page_start" in body: start, serr = _opt_int(body, "stated_page_start") if serr: diff --git a/backend/app/models/series_chapter.py b/backend/app/models/series_chapter.py index 4012174..9695272 100644 --- a/backend/app/models/series_chapter.py +++ b/backend/app/models/series_chapter.py @@ -12,6 +12,12 @@ A chapter may be a placeholder (is_placeholder=True) — a reserved empty slot f a section the operator doesn't have yet; it holds no pages and shows as a gap in the reader. stated_page_start/end carry the page range parsed from the source post (FC-6.2), used to flag missing-page gaps; both are nullable when unknown. + +stated_part is the operator-facing "Part N" label (FC-6.4), separate from the +positional chapter_number: chapter_number is auto-managed ordering (rewritten +1..N on reorder/delete), while stated_part is the real installment number the +operator types — e.g. a series authored from a post that is Part 2 of a story. +Nullable when unset (the UI then falls back to showing chapter_number). """ from datetime import datetime @@ -30,6 +36,7 @@ class SeriesChapter(Base): ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True ) chapter_number: Mapped[int] = mapped_column(Integer, nullable=False) + stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True) title: Mapped[str | None] = mapped_column(Text, nullable=True) is_placeholder: Mapped[bool] = mapped_column( Boolean, nullable=False, server_default="false" diff --git a/backend/app/models/tag_allowlist.py b/backend/app/models/tag_allowlist.py index 75bd0f2..3bbbc7a 100644 --- a/backend/app/models/tag_allowlist.py +++ b/backend/app/models/tag_allowlist.py @@ -22,7 +22,11 @@ class TagAllowlist(Base): tag_id: Mapped[int] = mapped_column( ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True ) - min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) + # Default auto-apply threshold for a newly-accepted tag. 0.90 (lowered from + # 0.95 on operator evidence 2026-06-07: 0.95 was too strict and skipped + # confident-enough applications). Per-tag value is still tunable in the + # allowlist table; existing rows keep whatever they were stored with. + min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.90) added_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/services/series_service.py b/backend/app/services/series_service.py index 7c943a1..9fbb634 100644 --- a/backend/app/services/series_service.py +++ b/backend/app/services/series_service.py @@ -131,6 +131,29 @@ class SeriesService: prev = ch return out + @staticmethod + def _part_gaps(chapters: list[dict]) -> list[dict]: + """Missing-Part gaps between consecutive chapters whose stated_part + numbers jump by more than 1 (e.g. a series with Part 1 and Part 3, or one + authored straight from a Part 2 post). Mirrors _gaps but on stated_part — + only chapters that actually carry a stated_part participate.""" + out: list[dict] = [] + prev = None + for ch in chapters: + cur = ch["stated_part"] + if cur is None: + continue + if prev is not None and cur > prev["stated_part"] + 1: + out.append( + { + "after_chapter_id": prev["id"], + "start": prev["stated_part"] + 1, + "end": cur - 1, + } + ) + prev = ch + return out + async def list_pages(self, series_tag_id: int) -> dict: tag = await self._require_series(series_tag_id) rows = ( @@ -138,6 +161,7 @@ class SeriesService: select( SeriesChapter.id.label("chapter_id"), SeriesChapter.chapter_number, + SeriesChapter.stated_part, SeriesChapter.title, SeriesChapter.is_placeholder, SeriesChapter.stated_page_start, @@ -149,10 +173,13 @@ class SeriesService: ImageRecord.mime, ImageRecord.path, ImageRecord.thumbnail_path, + ImageRecord.primary_post_id, + Post.post_title, ) .select_from(SeriesChapter) .outerjoin(SeriesPage, SeriesPage.chapter_id == SeriesChapter.id) .outerjoin(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .outerjoin(Post, Post.id == ImageRecord.primary_post_id) .where(SeriesChapter.series_tag_id == series_tag_id) .order_by( SeriesChapter.chapter_number.asc(), @@ -164,22 +191,30 @@ class SeriesService: chapters: list[dict] = [] flat: list[dict] = [] by_id: dict[int, dict] = {} + # chapter_id -> {post_id: title} seen across its pages, so we can label a + # chapter with its source post when all its pages come from one post. + posts_seen: dict[int, dict[int, str | None]] = {} for r in rows: ch = by_id.get(r.chapter_id) if ch is None: ch = { "id": r.chapter_id, "chapter_number": r.chapter_number, + "stated_part": r.stated_part, "title": r.title, "is_placeholder": r.is_placeholder, "stated_page_start": r.stated_page_start, "stated_page_end": r.stated_page_end, + "source_post": None, "pages": [], } by_id[r.chapter_id] = ch + posts_seen[r.chapter_id] = {} chapters.append(ch) if r.image_id is None: continue # placeholder / empty chapter + if r.primary_post_id is not None: + posts_seen[r.chapter_id][r.primary_post_id] = r.post_title page = { "image_id": r.image_id, "chapter_id": r.chapter_id, @@ -191,11 +226,21 @@ class SeriesService: ch["pages"].append(page) flat.append(page) + # A chapter's source_post is set only when every page shares one post — + # the common case (a series authored from a post). Mixed chapters stay + # null rather than guessing. + for ch in chapters: + seen = posts_seen.get(ch["id"], {}) + if len(seen) == 1: + pid, title = next(iter(seen.items())) + ch["source_post"] = {"id": pid, "title": title} + return { "series": {"id": tag.id, "name": tag.name}, "chapters": chapters, "pages": flat, # back-compat: flat reading order across chapters "gaps": self._gaps(chapters), + "part_gaps": self._part_gaps(chapters), } # ---- pages ------------------------------------------------------------ @@ -353,19 +398,23 @@ class SeriesService: chapter_id: int, *, title: str | None = None, + stated_part: int | None = None, stated_page_start: int | None = None, stated_page_end: int | None = None, set_title: bool = False, + set_part: bool = False, set_start: bool = False, set_end: bool = False, ) -> None: """Partial chapter edit. The set_* flags say which fields to write (so - None can be written explicitly, e.g. clearing a stated page).""" + None can be written explicitly, e.g. clearing a stated page or part).""" await self._require_series(series_tag_id) await self._require_chapter(series_tag_id, chapter_id) values: dict = {} if set_title: values["title"] = title + if set_part: + values["stated_part"] = stated_part if set_start: values["stated_page_start"] = stated_page_start if set_end: diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index c798bfb..4616812 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -743,6 +743,10 @@ async def normalize_existing_tags( "sample": sample, } start = time.monotonic() + log.info( + "normalize_existing_tags: %d group(s) need changes (budget=%ss)", + len(touched), time_budget_seconds, + ) for done, (key, members) in enumerate(touched): # Time-box: stop cleanly before the Celery limit kills us mid-group and # strands the run as a timeout. The caller re-enqueues to finish the @@ -754,6 +758,16 @@ async def normalize_existing_tags( summary["partial"] = True summary["remaining"] = len(touched) - done break + # Heartbeat so a long run is diagnosable instead of silent — the timeout + # operator-flagged 2026-06-07 produced zero logs because the only log was + # per finished group and it was stuck mid-group on a lock. + if done and done % 25 == 0: + log.info( + "normalize_existing_tags: %d/%d groups (%d merged, %d errors, " + "%.0fs elapsed)", + done, len(touched), summary["merged"], summary["errors"], + time.monotonic() - start, + ) canonical = key[2] names_by_id = dict(members) # Survivor: prefer a member already named canonically (no rename, no diff --git a/backend/app/tasks/_async_session.py b/backend/app/tasks/_async_session.py index e8c9094..0252dc3 100644 --- a/backend/app/tasks/_async_session.py +++ b/backend/app/tasks/_async_session.py @@ -15,8 +15,15 @@ from sqlalchemy.pool import NullPool from ..config import get_config -def async_session_factory(): - """Return ``(sessionmaker, engine)`` bound to a fresh async engine.""" +def async_session_factory(*, server_settings: dict | None = None): + """Return ``(sessionmaker, engine)`` bound to a fresh async engine. + + ``server_settings`` (optional) are applied by asyncpg as per-connection GUCs + on connect. Because NullPool opens a fresh real connection per checkout, every + transaction in the task inherits them — used to set ``lock_timeout`` so a + statement blocked on a lock fails fast instead of hanging to the Celery hard + limit (operator-flagged normalize_tags timeout 2026-06-07). + """ cfg = get_config() # NullPool: this engine lives for ONE task (created + disposed per # asyncio.run loop), so intra-task connection pooling buys nothing and @@ -26,5 +33,13 @@ def async_session_factory(): # phase 3 (asyncpg ConnectionDoesNotExistError, Anduo #40014). NullPool # opens a fresh real connection on each checkout, so phase 3 always # reconnects clean; pre_ping is then redundant. - engine = create_async_engine(cfg.database_url, future=True, poolclass=NullPool) + connect_args = {} + if server_settings: + connect_args["server_settings"] = { + k: str(v) for k, v in server_settings.items() + } + engine = create_async_engine( + cfg.database_url, future=True, poolclass=NullPool, + connect_args=connect_args, + ) return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 8096707..c6a1b08 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -126,7 +126,16 @@ def normalize_tags_task(self) -> dict: from ._async_session import async_session_factory async def _run() -> dict: - async_factory, async_engine = async_session_factory() + # lock_timeout=30s: a per-group merge repoints FKs across image_tag and + # series_page; if a statement blocks on a lock (e.g. behind a schema + # migration holding ACCESS EXCLUSIVE on series_page — the exact wedge that + # made this task run to the 40-min hard limit with no progress, + # operator-flagged 2026-06-07), it now fails fast. The per-group handler + # catches it (rollback + error++) and the loop continues, so one blocked + # group can't strand the whole chunk. + async_factory, async_engine = async_session_factory( + server_settings={"lock_timeout": "30s"} + ) try: async with async_factory() as session: # normalize_existing_tags commits per group internally. diff --git a/frontend/src/components/modal/FandomSetDialog.vue b/frontend/src/components/modal/FandomSetDialog.vue index d60a362..730bf2c 100644 --- a/frontend/src/components/modal/FandomSetDialog.vue +++ b/frontend/src/components/modal/FandomSetDialog.vue @@ -3,20 +3,29 @@ Fandom for “{{ tag.name }}”