Merge pull request 'feat(explore): auto-focus tag input on every image change' (#133) from dev into main

The workspace is built for rapid walk-and-tag, but the tag field was only
focused once (TagAutocomplete's on-mount autofocus) — walking to a neighbour
left focus behind, so the operator had to click the field each time
(operator-asked 2026-06-26).

TagPanel now exposes focusTagInput; ExploreView watches the focused image id and
re-focuses the field on seed + every walk via nextTick. Reuses the existing
focus path, so TagAutocomplete's mobile guard (no soft-keyboard pop on touch) is
preserved.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -69,3 +69,4 @@ Thumbs.db
 alembic/versions/__pycache__/
 *.sqlite
 *.sqlite-journal
+.superpowers/

Dockerfile

@@ -18,13 +18,16 @@ ENV PYTHONUNBUFFERED=1 \
 
 # System deps: ffmpeg (transcode + thumbnails, FC-2), unar (archives, FC-2),
 # libpq for psycopg, postgresql-client + zstd for FC-5 backup/restore
-# (pg_dump + tar --zstd), image libs.
+# (pg_dump + tar --zstd), image libs, megatools (mega.nz public-link downloads
+# for off-platform file-host links, #830 — `megatools dl`; Debian-native, no
+# external MEGA apt repo needed).
 RUN apt-get update && apt-get install -y --no-install-recommends \
         ffmpeg \
         unar \
         libpq5 \
         postgresql-client \
         zstd \
+        megatools \
         libjpeg62-turbo \
         libwebp7 \
         libpng16-16 \

alembic/env.py

@@ -1,13 +1,28 @@
 """Alembic environment — reads DATABASE_URL from app config."""
 
+import os
+import re
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config, pool
+from sqlalchemy import engine_from_config, pool, text
 
 from alembic import context
 from backend.app.config import get_config
 from backend.app.models import Base
 
+# Fail a blocked migration FAST instead of hanging forever. Migrations run
+# against the live DB while workers hold locks; 0040's `ALTER series_page` queued
+# behind a tag-merge that held a series_page lock for minutes (the merge runs an
+# unindexed full scan over image_record while repointing series_page) and hung
+# with no timeout — silent, indefinite (operator-flagged 2026-06-07). With a
+# lock_timeout a blocked DDL errors ("canceling statement due to lock timeout")
+# and the entrypoint's `alembic upgrade head` exits non-zero, so the deploy
+# retries / surfaces loudly rather than wedging. Override via env when a known
+# slow-lock window is expected.
+_MIGRATION_LOCK_TIMEOUT = os.environ.get("MIGRATION_LOCK_TIMEOUT", "30s")
+if not re.fullmatch(r"\d+\s*(ms|s|min)?", _MIGRATION_LOCK_TIMEOUT.strip()):
+    _MIGRATION_LOCK_TIMEOUT = "30s"  # ignore a malformed override
+
 config = context.config
 
 if config.config_file_name is not None:
@@ -38,6 +53,15 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
+        # Session-level lock_timeout for every DDL statement in this run. Set
+        # (and commit) before alembic opens its own transaction so the GUC
+        # persists on this connection regardless of how alembic structures its
+        # transactions. Value is from our own env, so f-string interpolation is
+        # safe (and it's been pattern-validated above); SET takes no bind params.
+        connection.execute(
+            text(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'")
+        )
+        connection.commit()
         context.configure(
             connection=connection,
             target_metadata=target_metadata,

alembic/versions/0037_patreon_seen_media.py

@@ -0,0 +1,53 @@
+"""patreon_seen_media: per-source ledger of already-ingested Patreon media
+
+Revision ID: 0037
+Revises: 0036
+Create Date: 2026-06-05
+
+Native Patreon ingester (build step 2a). Replaces gallery-dl's
+archive.sqlite3 with our own queryable table. The downloader upserts one
+row per (source, media) so routine walks skip media we've already
+processed; a future "recovery" mode bypasses the ledger to re-walk.
+
+`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
+``video:<post_id>:<media_id>`` — hence String(128). The unique
+constraint on (source_id, filehash) is the dedup upsert key.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0037"
+down_revision: Union[str, None] = "0036"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "patreon_seen_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("post_id", sa.String(64), nullable=True),
+        sa.Column(
+            "seen_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_seen_media_source_id"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("patreon_seen_media")

alembic/versions/0038_patreon_failed_media.py

@@ -0,0 +1,58 @@
+"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media
+
+Revision ID: 0038
+Revises: 0037
+Create Date: 2026-06-06
+
+Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN,
+deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here
+with an attempt counter; once it crosses the dead-letter threshold the ingester
+skips it on routine walks (recovery still re-attempts). A clean download clears
+the row. UNIQUE (source_id, filehash) is the upsert key (same media key the
+seen-ledger uses).
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0038"
+down_revision: Union[str, None] = "0037"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "patreon_failed_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("last_error", sa.Text, nullable=True),
+        sa.Column(
+            "first_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.Column(
+            "last_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_failed_media_source_id"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("patreon_failed_media")

alembic/versions/0039_library_audit_resume.py

@@ -0,0 +1,40 @@
+"""library_audit_run: resume cursor + progress timestamp for chunked scans
+
+Revision ID: 0039
+Revises: 0038
+Create Date: 2026-06-07
+
+scan_library_for_rule used to run one 2h pass that timed out on large libraries
+and monopolized the concurrency-1 maintenance queue (operator-flagged). It now
+runs short time-boxed chunks that re-enqueue: `resume_after_id` persists the
+keyset cursor so the next chunk continues where it left off, and
+`last_progress_at` lets the recovery sweep tell a progressing multi-chunk audit
+from a genuinely stuck one.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0039"
+down_revision: Union[str, None] = "0038"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "library_audit_run",
+        sa.Column(
+            "resume_after_id", sa.Integer, nullable=False, server_default="0"
+        ),
+    )
+    op.add_column(
+        "library_audit_run",
+        sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("library_audit_run", "last_progress_at")
+    op.drop_column("library_audit_run", "resume_after_id")

alembic/versions/0040_series_chapters.py

@@ -0,0 +1,108 @@
+"""series chapters: chapter layer over series_page (FC-6.1)
+
+Revision ID: 0040
+Revises: 0039
+Create Date: 2026-06-07
+
+A series (Tag kind='series') gains an ordered chapter layer. Reading order
+becomes (series_chapter.chapter_number, series_page.page_number). Every existing
+series is backfilled into a single auto-chapter (chapter_number=1) holding its
+current flat pages, so no data is lost and the old flat ordering is preserved.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0040"
+down_revision: Union[str, None] = "0039"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "series_chapter",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "series_tag_id",
+            sa.Integer,
+            sa.ForeignKey("tag.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("chapter_number", sa.Integer, nullable=False),
+        sa.Column("title", sa.Text, nullable=True),
+        sa.Column(
+            "is_placeholder", sa.Boolean, nullable=False, server_default="false"
+        ),
+        sa.Column("stated_page_start", sa.Integer, nullable=True),
+        sa.Column("stated_page_end", sa.Integer, nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+    )
+    op.create_index(
+        "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"]
+    )
+
+    # New columns on series_page; chapter_id starts nullable so we can backfill.
+    op.add_column(
+        "series_page", sa.Column("chapter_id", sa.Integer, nullable=True)
+    )
+    op.add_column(
+        "series_page", sa.Column("stated_page", sa.Integer, nullable=True)
+    )
+
+    conn = op.get_bind()
+    # One auto-chapter per existing series (any series_tag_id present in pages).
+    conn.execute(
+        sa.text(
+            "INSERT INTO series_chapter "
+            "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) "
+            "SELECT DISTINCT series_tag_id, 1, false, now(), now() "
+            "FROM series_page"
+        )
+    )
+    # Point every existing page at its series' auto-chapter.
+    conn.execute(
+        sa.text(
+            "UPDATE series_page sp "
+            "SET chapter_id = sc.id "
+            "FROM series_chapter sc "
+            "WHERE sc.series_tag_id = sp.series_tag_id"
+        )
+    )
+
+    # Now lock chapter_id down: NOT NULL + FK (cascade) + index.
+    op.alter_column("series_page", "chapter_id", nullable=False)
+    op.create_foreign_key(
+        "fk_series_page_chapter_id",
+        "series_page",
+        "series_chapter",
+        ["chapter_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_index(
+        "ix_series_page_chapter_id", "series_page", ["chapter_id"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_series_page_chapter_id", table_name="series_page")
+    op.drop_constraint(
+        "fk_series_page_chapter_id", "series_page", type_="foreignkey"
+    )
+    op.drop_column("series_page", "stated_page")
+    op.drop_column("series_page", "chapter_id")
+    op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter")
+    op.drop_table("series_chapter")

alembic/versions/0041_series_suggestions.py

@@ -0,0 +1,98 @@
+"""series suggestions: assisted-continuation matcher (FC-6.3)
+
+Revision ID: 0041
+Revises: 0040
+Create Date: 2026-06-07
+
+A confirm-only queue of "this post may continue this series" hints, plus two
+import_settings knobs (enable + score threshold) for the matcher.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0041"
+down_revision: Union[str, None] = "0040"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "series_suggestion",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "post_id",
+            sa.Integer,
+            sa.ForeignKey("post.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "series_tag_id",
+            sa.Integer,
+            sa.ForeignKey("tag.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("score", sa.Float, nullable=False),
+        sa.Column("signals", sa.JSON, nullable=True),
+        sa.Column(
+            "status", sa.String(16), nullable=False, server_default="pending"
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+        sa.UniqueConstraint(
+            "post_id", "series_tag_id", name="uq_series_suggestion_post_series"
+        ),
+    )
+    op.create_index(
+        "ix_series_suggestion_post_id", "series_suggestion", ["post_id"]
+    )
+    op.create_index(
+        "ix_series_suggestion_series_tag_id",
+        "series_suggestion",
+        ["series_tag_id"],
+    )
+    op.create_index(
+        "ix_series_suggestion_status", "series_suggestion", ["status"]
+    )
+
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "series_suggest_enabled",
+            sa.Boolean,
+            nullable=False,
+            server_default=sa.true(),
+        ),
+    )
+    op.add_column(
+        "import_settings",
+        sa.Column(
+            "series_suggest_threshold",
+            sa.Float,
+            nullable=False,
+            server_default="0.5",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("import_settings", "series_suggest_threshold")
+    op.drop_column("import_settings", "series_suggest_enabled")
+    op.drop_index("ix_series_suggestion_status", table_name="series_suggestion")
+    op.drop_index(
+        "ix_series_suggestion_series_tag_id", table_name="series_suggestion"
+    )
+    op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion")
+    op.drop_table("series_suggestion")

alembic/versions/0042_series_chapter_stated_part.py

@@ -0,0 +1,32 @@
+"""series chapter stated_part: operator-facing Part N label (FC-6.4)
+
+Revision ID: 0042
+Revises: 0041
+Create Date: 2026-06-07
+
+A chapter's positional chapter_number is auto-managed (rewritten 1..N on
+reorder/delete), so it can't double as the installment number the operator wants
+to type (e.g. a series authored from a post that is Part 2). Add a nullable
+stated_part alongside it — the same split as series_page.page_number (order) vs
+series_page.stated_page (printed number). Nullable; the UI falls back to
+chapter_number when unset.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0042"
+down_revision: Union[str, None] = "0041"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "series_chapter", sa.Column("stated_part", sa.Integer, nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("series_chapter", "stated_part")

alembic/versions/0043_post_attachment_per_post_unique.py

@@ -0,0 +1,62 @@
+"""post_attachment: per-post sha uniqueness (empty-post flood fix)
+
+Revision ID: 0043
+Revises: 0042
+Create Date: 2026-06-08
+
+PostAttachment.sha256 was GLOBALLY unique, so a non-art file the creator attaches
+to many posts (a standard pdf/zip/link-card) only ever got ONE row — on the first
+post — leaving every later post a bare shell (no image, no attachment). The native
+Patreon backfill of Anduo surfaced 1589 such shells (operator-flagged 2026-06-08).
+
+Switch to PER-POST uniqueness: the on-disk blob stays sha-deduped, but each post
+gets its own row. Replace the unique sha256 index with a plain lookup index plus
+two partial uniques — (post_id, sha256) for real posts and (sha256) for the
+NULL-post filesystem case (still one row per file there).
+
+Existing data has ≤1 row per sha (the old global unique), so the new partial
+uniques can't be violated on upgrade — no data backfill needed here. The bare-post
+shells themselves are removed by the separate prune-empty-posts cleanup tool.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0043"
+down_revision: Union[str, None] = "0042"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Drop the global unique index; recreate it as a plain (non-unique) lookup
+    # index so sha-based reads keep their index (matches the model's index=True).
+    op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
+    op.create_index(
+        "ix_post_attachment_sha256", "post_attachment", ["sha256"],
+    )
+    op.create_index(
+        "uq_post_attachment_post_sha", "post_attachment",
+        ["post_id", "sha256"], unique=True,
+        postgresql_where=sa.text("post_id IS NOT NULL"),
+    )
+    op.create_index(
+        "uq_post_attachment_null_post_sha", "post_attachment",
+        ["sha256"], unique=True,
+        postgresql_where=sa.text("post_id IS NULL"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "uq_post_attachment_null_post_sha", table_name="post_attachment"
+    )
+    op.drop_index(
+        "uq_post_attachment_post_sha", table_name="post_attachment"
+    )
+    op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
+    op.create_index(
+        "ix_post_attachment_sha256", "post_attachment", ["sha256"],
+        unique=True,
+    )

alembic/versions/0044_ml_settings_tagger_store_floor.py

@@ -0,0 +1,37 @@
+"""ml_settings.tagger_store_floor
+
+The ingest confidence floor below which tagger predictions are not stored,
+promoted from the TAGGER_STORE_FLOOR env var to a DB-backed, UI-tunable
+setting. Default 0.70 (was an env default of 0.05): the suggestion path
+already filters at 0.70 and the centroid/learned path covers low-confidence
+preferred tags, so the sub-0.70 tail was redundant weight — it had grown
+image_record's TOAST to ~100 GB. See plan-task #764.
+
+Revision ID: 0044
+Revises: 0043
+Create Date: 2026-06-10
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0044"
+down_revision: Union[str, None] = "0043"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "ml_settings",
+        sa.Column(
+            "tagger_store_floor", sa.Float(),
+            nullable=False, server_default="0.7",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("ml_settings", "tagger_store_floor")

alembic/versions/0045_image_prediction_table.py

@@ -0,0 +1,69 @@
+"""image_prediction table (DDL only — backfill runs as a background task)
+
+Normalizes the per-image tagger predictions out of the JSON blob into a
+queryable table (#768). This migration creates ONLY the table + indexes — it
+is pure DDL and commits instantly, so web boots immediately.
+
+The data backfill from the existing image_record.tagger_predictions JSON is
+deliberately NOT done here. Doing it inline made the whole migration one
+transaction over the ~100 GB TOAST: nothing committed until the very end, it
+was invisible/unmonitorable mid-run, and an early MATERIALIZED-CTE form spilled
+the full 100 GB to temp. Instead the backfill is the
+backend.app.tasks.admin.backfill_image_predictions_task — batched by id window,
+committed per chunk (visible progress + resumable), idempotent
+(ON CONFLICT DO NOTHING). Trigger it from Settings → Maintenance once web is up.
+
+The old image_record.tagger_predictions column is left in place (vestigial) and
+dropped in a follow-up once the backfill + code cutover are verified — dropping
+it needs an ACCESS EXCLUSIVE lock on the hot image_record table (the 0044 lock
+class), so it's deferred to a quiesced-worker window.
+
+Revision ID: 0045
+Revises: 0044
+Create Date: 2026-06-10
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0045"
+down_revision: Union[str, None] = "0044"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "image_prediction",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "image_record_id", sa.Integer(),
+            sa.ForeignKey("image_record.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("raw_name", sa.String(length=255), nullable=False),
+        sa.Column("category", sa.String(length=64), nullable=False),
+        sa.Column("score", sa.Float(), nullable=False),
+        sa.UniqueConstraint(
+            "image_record_id", "raw_name", name="image_raw_name",
+        ),
+    )
+    op.create_index(
+        "ix_image_prediction_image", "image_prediction", ["image_record_id"],
+    )
+    op.create_index(
+        "ix_image_prediction_name_score", "image_prediction",
+        ["raw_name", "score"],
+    )
+    # No data backfill here — see the module docstring. The one-time copy from
+    # image_record.tagger_predictions runs as backfill_image_predictions_task
+    # (batched, resumable, idempotent), kept out of this transaction so web boots
+    # without waiting on a ~100 GB pass.
+
+
+def downgrade() -> None:
+    op.drop_index("ix_image_prediction_name_score", "image_prediction")
+    op.drop_index("ix_image_prediction_image", "image_prediction")
+    op.drop_table("image_prediction")

alembic/versions/0046_drop_tagger_predictions.py

@@ -0,0 +1,43 @@
+"""drop image_record.tagger_predictions (predictions normalized to image_prediction)
+
+Final step of #768. The per-tag predictions now live in the image_prediction
+table (backfilled from the JSON, read by suggestions + allowlist, written by
+tag_and_embed). The old JSON column is dead weight — and it's the ~100 GB of
+sub-0.70 score tail that bloated image_record's TOAST and broke DB backups
+(#739). Dropping it is a fast catalog change; it does NOT reclaim the disk on
+its own — run `VACUUM FULL image_record` (or pg_repack) afterward, off-hours,
+to return the space to the OS so backups go small.
+
+DROP COLUMN needs a brief ACCESS EXCLUSIVE lock on image_record; env.py's
+lock_timeout guards it, so quiesce the ml-worker if a tagging run is in flight
+(see the migration-lock reference). tagger_model_version is kept — it's the
+"has this been tagged / is it current?" signal the backfill sweep reads.
+
+Revision ID: 0046
+Revises: 0045
+Create Date: 2026-06-11
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0046"
+down_revision: Union[str, None] = "0045"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.drop_column("image_record", "tagger_predictions")
+
+
+def downgrade() -> None:
+    # Re-add the column empty. The JSON data is not restored (it lived only in
+    # this column); a downgrade would re-tag or backfill from image_prediction
+    # separately if ever needed.
+    op.add_column(
+        "image_record",
+        sa.Column("tagger_predictions", sa.JSON(), nullable=True),
+    )

alembic/versions/0047_series_chapter_dividers.py

@@ -0,0 +1,175 @@
+"""series chapters become cosmetic dividers; pages become one series-global run
+
+FC-6.x reframe (#789). A series is now ONE flat, series-global ordered run of
+pages; chapters stop owning pages and become labeled dividers anchored to the
+page that begins them.
+
+Migration (order matters — series_page.chapter_id cascades, so it must be
+dropped BEFORE any chapter row is deleted, or pages would cascade away):
+  a. Renumber series_page.page_number to a series-global 1..N (ordered by the
+     OLD (chapter_number, page_number)).
+  b. Add series_chapter.anchor_page_id and populate it with each chapter's first
+     page (lowest new page_number).
+  c. Drop series_page.chapter_id (severs the cascade link).
+  d. Prune chapters that shouldn't become dividers: empty/placeholder ones (no
+     anchor) and the redundant unlabeled chapter that would sit at page 1.
+  e. Reshape series_chapter into the divider: drop chapter_number,
+     is_placeholder, stated_page_start/end; make anchor_page_id NOT NULL +
+     UNIQUE + FK→series_page ON DELETE CASCADE.
+
+Revision ID: 0047
+Revises: 0046
+Create Date: 2026-06-11
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0047"
+down_revision: Union[str, None] = "0046"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # a. series-global page numbering, preserving the old reading order.
+    op.execute(
+        """
+        WITH ordered AS (
+            SELECT sp.id,
+                   ROW_NUMBER() OVER (
+                       PARTITION BY sp.series_tag_id
+                       ORDER BY sc.chapter_number, sp.page_number, sp.id
+                   ) AS rn
+            FROM series_page sp
+            JOIN series_chapter sc ON sc.id = sp.chapter_id
+        )
+        UPDATE series_page sp
+        SET page_number = ordered.rn
+        FROM ordered
+        WHERE sp.id = ordered.id
+        """
+    )
+
+    # b. anchor each existing chapter at its first page (lowest new page_number).
+    op.add_column(
+        "series_chapter",
+        sa.Column("anchor_page_id", sa.Integer(), nullable=True),
+    )
+    op.execute(
+        """
+        WITH firsts AS (
+            SELECT DISTINCT ON (sp.chapter_id)
+                   sp.chapter_id, sp.id AS page_id
+            FROM series_page sp
+            ORDER BY sp.chapter_id, sp.page_number, sp.id
+        )
+        UPDATE series_chapter sc
+        SET anchor_page_id = firsts.page_id
+        FROM firsts
+        WHERE firsts.chapter_id = sc.id
+        """
+    )
+
+    # c. sever the ownership link (drops the FK + index with the column) BEFORE
+    #    pruning chapters, so deleting a chapter can't cascade-delete its pages.
+    op.drop_column("series_page", "chapter_id")
+
+    # d. prune chapters that don't become dividers: placeholders / empty ones
+    #    (no anchor), and the unlabeled chapter that would land redundantly at
+    #    page 1 (the series just starts — no divider needed there).
+    op.execute(
+        """
+        DELETE FROM series_chapter sc
+        USING (
+            SELECT sc2.id
+            FROM series_chapter sc2
+            LEFT JOIN series_page sp ON sp.id = sc2.anchor_page_id
+            WHERE sc2.anchor_page_id IS NULL
+               OR (sp.page_number = 1
+                   AND sc2.title IS NULL
+                   AND sc2.stated_part IS NULL)
+        ) gone
+        WHERE sc.id = gone.id
+        """
+    )
+
+    # e. reshape into the divider model.
+    op.drop_column("series_chapter", "chapter_number")
+    op.drop_column("series_chapter", "is_placeholder")
+    op.drop_column("series_chapter", "stated_page_start")
+    op.drop_column("series_chapter", "stated_page_end")
+    op.alter_column("series_chapter", "anchor_page_id", nullable=False)
+    op.create_unique_constraint(
+        "uq_series_chapter_anchor_page", "series_chapter", ["anchor_page_id"]
+    )
+    op.create_foreign_key(
+        "fk_series_chapter_anchor_page",
+        "series_chapter",
+        "series_page",
+        ["anchor_page_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+
+
+def downgrade() -> None:
+    # Lossy: dividers can't be reconstructed as owning chapters. Collapse back to
+    # exactly one chapter per series that owns all its pages in order.
+    op.add_column(
+        "series_page", sa.Column("chapter_id", sa.Integer(), nullable=True)
+    )
+    op.drop_constraint(
+        "fk_series_chapter_anchor_page", "series_chapter", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "uq_series_chapter_anchor_page", "series_chapter", type_="unique"
+    )
+    op.drop_column("series_chapter", "anchor_page_id")
+    op.add_column(
+        "series_chapter",
+        sa.Column(
+            "chapter_number", sa.Integer(), nullable=False, server_default="1"
+        ),
+    )
+    op.add_column(
+        "series_chapter",
+        sa.Column(
+            "is_placeholder", sa.Boolean(), nullable=False,
+            server_default="false",
+        ),
+    )
+    op.add_column(
+        "series_chapter",
+        sa.Column("stated_page_start", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "series_chapter",
+        sa.Column("stated_page_end", sa.Integer(), nullable=True),
+    )
+    op.execute("DELETE FROM series_chapter")
+    op.execute(
+        """
+        INSERT INTO series_chapter (series_tag_id, chapter_number)
+        SELECT DISTINCT series_tag_id, 1 FROM series_page
+        """
+    )
+    op.execute(
+        """
+        UPDATE series_page sp
+        SET chapter_id = sc.id
+        FROM series_chapter sc
+        WHERE sc.series_tag_id = sp.series_tag_id
+        """
+    )
+    op.alter_column("series_page", "chapter_id", nullable=False)
+    op.create_foreign_key(
+        "fk_series_page_chapter",
+        "series_page",
+        "series_chapter",
+        ["chapter_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )

alembic/versions/0048_series_page_pending_status.py

@@ -0,0 +1,45 @@
+"""series_page pending staging: status + nullable page_number (#789 Phase 2)
+
+Pages added from a post no longer append straight into the run — they land
+'pending' with a NULL page_number, staged grouped by their source post so the
+operator can drop junk (text-free alts, bumpers) and place the keepers into the
+sequence. A page only gets a series-global page_number once it's 'placed'.
+
+Revision ID: 0048
+Revises: 0047
+Create Date: 2026-06-11
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0048"
+down_revision: Union[str, None] = "0047"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "series_page",
+        sa.Column(
+            "status", sa.String(length=16), nullable=False,
+            server_default="placed",
+        ),
+    )
+    op.alter_column(
+        "series_page", "page_number",
+        existing_type=sa.Integer(), nullable=True,
+    )
+
+
+def downgrade() -> None:
+    # Lossy: pending pages are unsorted staging rows with no order — drop them.
+    op.execute("DELETE FROM series_page WHERE status = 'pending'")
+    op.alter_column(
+        "series_page", "page_number",
+        existing_type=sa.Integer(), nullable=False,
+    )
+    op.drop_column("series_page", "status")

alembic/versions/0049_external_link_table.py

@@ -0,0 +1,90 @@
+"""external_link table — off-platform file-host links found in post bodies
+
+Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox /
+Pixeldrain and link them in the post text. This table records each such link
+(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger
+the download worker (a later slice) walks. `url` keeps the FULL link including
+the `#fragment` — mega.nz's decryption key lives there; truncating it makes the
+file undownloadable.
+
+CHECK whitelists for host + status include the full enum up front (incl. the
+download-worker statuses) so the worker slice needs no constraint migration.
+
+Revision ID: 0049
+Revises: 0048
+Create Date: 2026-06-14
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0049"
+down_revision: Union[str, None] = "0048"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "external_link",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "post_id", sa.Integer(),
+            sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False,
+        ),
+        sa.Column(
+            "artist_id", sa.Integer(),
+            sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
+        ),
+        sa.Column("host", sa.String(length=16), nullable=False),
+        sa.Column("url", sa.Text(), nullable=False),
+        sa.Column("label", sa.Text(), nullable=True),
+        sa.Column(
+            "status", sa.String(length=16), nullable=False,
+            server_default="pending",
+        ),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_error", sa.Text(), nullable=True),
+        sa.Column(
+            "attachment_id", sa.Integer(),
+            sa.ForeignKey("post_attachment.id", ondelete="SET NULL"),
+            nullable=True,
+        ),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True), nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("duration_seconds", sa.Float(), nullable=True),
+        sa.CheckConstraint(
+            "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')",
+            name="ck_external_link_host",
+        ),
+        sa.CheckConstraint(
+            "status IN ('pending','downloading','downloaded','failed',"
+            "'skipped','dead')",
+            name="ck_external_link_status",
+        ),
+    )
+    op.create_index(
+        "ix_external_link_post_id", "external_link", ["post_id"],
+    )
+    op.create_index(
+        "ix_external_link_artist_id", "external_link", ["artist_id"],
+    )
+    op.create_index(
+        "ix_external_link_status", "external_link", ["status"],
+    )
+    op.create_index(
+        "uq_external_link_post_url", "external_link", ["post_id", "url"],
+        unique=True,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("uq_external_link_post_url", table_name="external_link")
+    op.drop_index("ix_external_link_status", table_name="external_link")
+    op.drop_index("ix_external_link_artist_id", table_name="external_link")
+    op.drop_index("ix_external_link_post_id", table_name="external_link")
+    op.drop_table("external_link")

alembic/versions/0050_external_link_host_toggles.py

@@ -0,0 +1,38 @@
+"""import_settings: per-host enable toggles for external file-host downloads
+
+Operator levers (#830): disable a single host (e.g. mega.nz when it's
+rate-limiting/banning) without touching the others. The worker reads these via
+getattr and defaults to enabled, so the toggles default TRUE (works out of the
+box, rule #26).
+
+Revision ID: 0050
+Revises: 0049
+Create Date: 2026-06-14
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0050"
+down_revision: Union[str, None] = "0049"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
+
+
+def upgrade() -> None:
+    for host in _HOSTS:
+        op.add_column(
+            "import_settings",
+            sa.Column(
+                f"extdl_{host}_enabled", sa.Boolean(), nullable=False,
+                server_default=sa.true(),
+            ),
+        )
+
+
+def downgrade() -> None:
+    for host in _HOSTS:
+        op.drop_column("import_settings", f"extdl_{host}_enabled")

alembic/versions/0051_image_source_provenance.py

@@ -0,0 +1,38 @@
+"""image_record: source_url + source_filehash (inline-image localization)
+
+#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline
+images instead of hotlinking the public CDN. The join key between a body
+`<img src=CDN>` and the local file is the CDN's 32-hex filehash (the same
+identity extract_media dedups by). Persist it (indexed) plus the full source
+URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing
+rows — those fall back to hotlinking until re-downloaded.
+
+Revision ID: 0051
+Revises: 0050
+Create Date: 2026-06-14
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0051"
+down_revision: Union[str, None] = "0050"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True))
+    op.add_column(
+        "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True)
+    )
+    op.create_index(
+        "ix_image_record_source_filehash", "image_record", ["source_filehash"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_image_record_source_filehash", table_name="image_record")
+    op.drop_column("image_record", "source_filehash")
+    op.drop_column("image_record", "source_url")

alembic/versions/0052_image_duration_seconds.py

@@ -0,0 +1,32 @@
+"""image_record: duration_seconds (Tier-1 video near-dup key)
+
+#871. Videos previously deduped on sha256 only (pHash is images-only), so a
+different encode/remux of the same video imported as a distinct record. Persist
+the container duration so the importer can treat same-artist videos with matching
+duration (+ aspect ratio) as the same content and dedup/supersede like images.
+NULL for images and for video rows imported before this column existed (a
+backfill re-probes those so they participate in dedup).
+
+Revision ID: 0052
+Revises: 0051
+Create Date: 2026-06-16
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0052"
+down_revision: Union[str, None] = "0051"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("image_record", "duration_seconds")

alembic/versions/0053_ml_settings_video_tagging.py

@@ -0,0 +1,49 @@
+"""ml_settings: video tagging knobs (cadence sampling + noise floor)
+
+#747. Video tag quality/perf: sample frames at a fixed cadence (interval) so a
+tag's frame-presence reflects real screen time, cap total frames so long videos
+stay bounded, and keep a tag only if it appears in >= min_tag_frames sampled
+frames. Operator-tunable via Settings → ML (replaces the VIDEO_ML_FRAMES env var).
+
+Revision ID: 0053
+Revises: 0052
+Create Date: 2026-06-16
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0053"
+down_revision: Union[str, None] = "0052"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "ml_settings",
+        sa.Column(
+            "video_frame_interval_seconds", sa.Float(), nullable=False,
+            server_default="4.0",
+        ),
+    )
+    op.add_column(
+        "ml_settings",
+        sa.Column(
+            "video_max_frames", sa.Integer(), nullable=False, server_default="64",
+        ),
+    )
+    op.add_column(
+        "ml_settings",
+        sa.Column(
+            "video_min_tag_frames", sa.Integer(), nullable=False,
+            server_default="3",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("ml_settings", "video_min_tag_frames")
+    op.drop_column("ml_settings", "video_max_frames")
+    op.drop_column("ml_settings", "video_frame_interval_seconds")

alembic/versions/0054_subscribestar_ledgers.py

@@ -0,0 +1,82 @@
+"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers
+
+Revision ID: 0054
+Revises: 0053
+Create Date: 2026-06-17
+
+SubscribeStar native ingester (phase 1 of the gallery-dl → native-core
+migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so
+routine walks skip already-ingested media (recovery bypasses it) and a
+dead-letter ledger so persistently-failing media stops re-burning backfill
+chunks. `filehash` is a CDN content hash when present, else a synthesized
+``<post_id>:<filename>`` key — hence String(128). UNIQUE (source_id, filehash)
+is the upsert key on each.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0054"
+down_revision: Union[str, None] = "0053"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "subscribestar_seen_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("post_id", sa.String(64), nullable=True),
+        sa.Column(
+            "seen_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_subscribestar_seen_media_source_id"
+        ),
+    )
+    op.create_table(
+        "subscribestar_failed_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("last_error", sa.Text, nullable=True),
+        sa.Column(
+            "first_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.Column(
+            "last_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_subscribestar_failed_media_source_id"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("subscribestar_failed_media")
+    op.drop_table("subscribestar_seen_media")

alembic/versions/0055_image_provenance_from_attachment.py

@@ -0,0 +1,55 @@
+"""image_provenance: from_attachment_id (which archive an image was extracted from)
+
+Milestone #87. When an image is pulled out of a .zip/.rar, record WHICH archive
+PostAttachment it came from, so the provenance UI can show the single archive a
+file lives inside instead of every attachment on the post. Nullable FK with
+ON DELETE SET NULL — a loose (non-archive) download leaves it NULL, and deleting
+the archive attachment forgets the linkage without destroying the (image, post)
+provenance edge. Existing rows are NULL until the reextract backfill stamps them.
+
+Revision ID: 0055
+Revises: 0054
+Create Date: 2026-06-22
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0055"
+down_revision: Union[str, None] = "0054"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "image_provenance",
+        sa.Column("from_attachment_id", sa.Integer(), nullable=True),
+    )
+    op.create_index(
+        "ix_image_provenance_from_attachment_id",
+        "image_provenance",
+        ["from_attachment_id"],
+    )
+    op.create_foreign_key(
+        "fk_image_provenance_from_attachment",
+        "image_provenance",
+        "post_attachment",
+        ["from_attachment_id"],
+        ["id"],
+        ondelete="SET NULL",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "fk_image_provenance_from_attachment",
+        "image_provenance",
+        type_="foreignkey",
+    )
+    op.drop_index(
+        "ix_image_provenance_from_attachment_id",
+        table_name="image_provenance",
+    )
+    op.drop_column("image_provenance", "from_attachment_id")

backend/app/api/admin.py

@@ -6,6 +6,7 @@ Five action surfaces:
   DELETE /api/admin/tags/<int:tag_id>              (Tier B)
   POST   /api/admin/tags/<int:dest_id>/merge       (Tier B)
   POST   /api/admin/tags/prune-unused              (Tier A)
+  POST   /api/admin/posts/prune-bare               (Tier A)
   POST   /api/admin/tags/purge-legacy              (Tier A)
   GET    /api/admin/tags/<int:tag_id>/usage-count  (helper)
 
@@ -38,6 +39,31 @@ def _bulk_image_confirm_token(image_ids: list[int]) -> str:
     return digest[:8]
 
 
+async def _run_dry_run_op(service_fn, **service_kwargs):
+    """Shared body for the Tier-A dry-run/apply endpoints: read the `dry_run`
+    flag, run the cleanup_service predicate under `run_sync`, and return its
+    result dict. The SAME `service_fn` drives both preview and apply (the flag
+    just toggles), so a handler physically can't let its preview diverge from
+    its delete (rule 93). Default False preserves the existing contract — the UI
+    always passes `dry_run` explicitly (true to preview, false to apply). Extra
+    service kwargs (e.g. `source_id`) pass straight through."""
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", False))
+    async with get_session() as session:
+        result = await session.run_sync(
+            lambda sync_sess: service_fn(sync_sess, dry_run=dry_run, **service_kwargs)
+        )
+    return jsonify(result)
+
+
+def _queued(async_result):
+    """Standard 202 for an operator-triggered maintenance task: hand the UI the
+    Celery task id so it can tail /maintenance/task-result (or the activity
+    dashboard) for the summary. (trigger_vacuum stays bespoke — the UI doesn't
+    poll it, so it returns no task id.)"""
+    return jsonify({"task_id": async_result.id, "status": "queued"}), 202
+
+
 @admin_bp.route("/artists/<slug>/cascade-delete", methods=["POST"])
 async def artist_cascade_delete(slug: str):
     body = await request.get_json(silent=True) or {}
@@ -145,6 +171,30 @@ async def tag_merge(dest_id: int):
     if not isinstance(source_id, int) or source_id == dest_id:
         return _bad("invalid_source_id", detail="source_id must be int and differ from dest")
 
+    # dry_run: non-mutating preview (counts + sample) so the operator can
+    # confirm the target before the irreversible merge (#8, rule 93 parity).
+    if body.get("dry_run"):
+        async with get_session() as session:
+            try:
+                p = await TagService(session).merge_preview(
+                    source_id=source_id, target_id=dest_id,
+                )
+            except TagValidationError as exc:
+                return _bad("tag_not_found", status=404, detail=str(exc))
+        return jsonify({
+            "preview": {
+                "source_id": p.source_id, "source_name": p.source_name,
+                "target_id": p.target_id, "target_name": p.target_name,
+                "compatible": p.compatible,
+                "images_moving": p.images_moving,
+                "images_already_on_target": p.images_already_on_target,
+                "source_total": p.source_total,
+                "series_pages": p.series_pages,
+                "will_alias": p.will_alias,
+                "sample_thumbnails": p.sample_thumbnails,
+            },
+        })
+
     async with get_session() as session:
         try:
             result = await TagService(session).merge(
@@ -192,16 +242,39 @@ async def tags_prune_unused():
     re-call with dry_run=false."""
     from ..services.cleanup_service import prune_unused_tags
 
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", False))
+    return await _run_dry_run_op(prune_unused_tags)
 
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda sync_sess: prune_unused_tags(
-                sync_sess, dry_run=dry_run,
-            )
-        )
-    return jsonify(result)
+
+@admin_bp.route("/posts/prune-bare", methods=["POST"])
+async def posts_prune_bare():
+    """Tier-A: delete bare posts — Post rows with no linked images (primary OR
+    provenance) and no attachments. Dry-run preview list IS the prompt: UI calls
+    with dry_run=true first, shows the count + sample, operator confirms by
+    re-calling with dry_run=false. Same preview/apply-parity predicate as the
+    prune itself, so the preview can't diverge from the delete."""
+    from ..services.cleanup_service import prune_bare_posts
+
+    return await _run_dry_run_op(prune_bare_posts)
+
+
+@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"])
+async def posts_reconcile_duplicates():
+    """Tier-A: unify duplicate post rows for the same real post — the gallery-dl
+    (attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper,
+    moving image/provenance/attachment/link rows over. Images are untouched.
+    dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies
+    and returns {groups, merged, sample}. Optional source_id scopes to one source.
+    Same find_duplicate_post_groups predicate drives preview + apply (rule 93)."""
+    from ..services.cleanup_service import reconcile_duplicate_posts
+
+    body = await request.get_json(silent=True) or {}
+    raw_source = body.get("source_id")
+    try:
+        source_id = int(raw_source) if raw_source is not None else None
+    except (TypeError, ValueError):
+        return _bad("invalid_source_id", detail="source_id must be an integer")
+
+    return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
 
 
 @admin_bp.route("/tags/purge-legacy", methods=["POST"])
@@ -214,14 +287,7 @@ async def tags_purge_legacy():
     operator confirms with dry_run=false."""
     from ..services.cleanup_service import purge_legacy_tags
 
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", False))
-
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda sync_sess: purge_legacy_tags(sync_sess, dry_run=dry_run)
-        )
-    return jsonify(result)
+    return await _run_dry_run_op(purge_legacy_tags)
 
 
 @admin_bp.route("/tags/reset-content", methods=["POST"])
@@ -229,20 +295,39 @@ async def tags_reset_content():
     """Tier-A: delete ALL general + character tags (the Camie-suggestable
     content vocabulary) so the operator can re-tag from scratch via
     auto-suggest. fandom + series tags + series_page ordering are preserved,
-    and image tagger_predictions are untouched so suggestions repopulate.
+    and image_prediction rows are untouched so suggestions repopulate.
     dry-run preview returns per-kind counts + applications + a sample so the
     UI shows exactly what'll go before the operator confirms (dry_run=false).
     Irreversible except via DB backup restore."""
     from ..services.cleanup_service import reset_content_tagging
 
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", False))
+    return await _run_dry_run_op(reset_content_tagging)
 
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda sync_sess: reset_content_tagging(sync_sess, dry_run=dry_run)
-        )
-    return jsonify(result)
+
+@admin_bp.route("/tags/normalize", methods=["POST"])
+async def tags_normalize():
+    """#714: retro-normalize existing tags to the #701 canonical form (Title
+    Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
+
+    dry_run=true (default) returns a projection inline — group/collision/rename
+    counts + a sample of the changes — so the UI shows exactly what'll happen.
+    dry_run=false dispatches the long-running maintenance task (the merge FK
+    repoints can touch many tags); the UI tails the activity dashboard for the
+    summary. Idempotent; back up first (the merges are irreversible)."""
+    from ..services.tag_service import normalize_existing_tags
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))
+
+    if dry_run:
+        async with get_session() as session:
+            result = await normalize_existing_tags(session, dry_run=True)
+        return jsonify(result)
+
+    from ..tasks.admin import normalize_tags_task
+
+    async_result = normalize_tags_task.delay()
+    return _queued(async_result)
 
 
 @admin_bp.route("/maintenance/db-stats", methods=["GET"])
@@ -289,3 +374,74 @@ async def trigger_vacuum():
 
     vacuum_analyze.delay()
     return jsonify({"status": "queued"}), 202
+
+
+@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
+async def trigger_reextract_archives():
+    """Operator-triggered re-extract (#713): PostAttachments that are actually
+    archives but were filed opaquely (pre magic-byte gate) get extracted and
+    their members linked to the post. Idempotent; runs on the maintenance queue."""
+    from ..tasks.admin import reextract_archive_attachments_task
+
+    async_result = reextract_archive_attachments_task.delay()
+    return _queued(async_result)
+
+
+@admin_bp.route("/maintenance/prune-missing-files", methods=["POST"])
+async def trigger_prune_missing_files():
+    """Operator-triggered orphan repair (#859): delete ImageRecords whose backing
+    file is gone from disk (e.g. left by the external-attach unlink bug), so they
+    stop 404-ing on playback. The task aborts WITHOUT deleting if a large fraction
+    of files look missing (a filesystem/NFS stall). Maintenance queue;
+    operator-triggered only — never an unattended sweep."""
+    from ..tasks.admin import prune_missing_file_records_task
+
+    async_result = prune_missing_file_records_task.delay()
+    return _queued(async_result)
+
+
+@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
+async def trigger_dedup_videos():
+    """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
+    what would be removed (groups / redundant count / reclaimable bytes) WITHOUT
+    deleting; dry_run=false applies it (re-link posts to the keeper, then delete
+    the redundant copies). Either way it first re-probes NULL-duration videos so
+    the existing library participates. Returns the Celery task id — poll
+    /maintenance/task-result/<id> for the summary."""
+    from ..tasks.admin import dedup_videos_task
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))  # default to the SAFE preview
+    async_result = dedup_videos_task.delay(dry_run=dry_run)
+    return _queued(async_result)
+
+
+@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"])
+async def trigger_purge_gated_previews():
+    """Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how
+    many blurred locked-preview images (grabbed from tier-gated Patreon posts
+    before the fix) would be removed WITHOUT deleting; dry_run=false applies it.
+    Re-walks every enabled Patreon source read-only and matches by content hash, so
+    real content downloaded when access existed is provably spared. Returns the
+    Celery task id — poll /maintenance/task-result/<id> for the summary."""
+    from ..tasks.admin import purge_gated_previews_task
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))  # default to the SAFE preview
+    async_result = purge_gated_previews_task.delay(dry_run=dry_run)
+    return _queued(async_result)
+
+
+@admin_bp.route("/maintenance/task-result/<task_id>", methods=["GET"])
+async def maintenance_task_result(task_id: str):
+    """Poll a maintenance Celery task's result (the summary dict it returns).
+    Used by the video-dedup card to show the dry-run projection before apply."""
+    from ..celery_app import celery
+
+    res = celery.AsyncResult(task_id)
+    ready = res.ready()
+    return jsonify({
+        "ready": ready,
+        "successful": res.successful() if ready else None,
+        "result": res.result if (ready and res.successful()) else None,
+    })

backend/app/api/allowlist.py

@@ -20,12 +20,37 @@ async def list_allowlist():
                 "tag_name": r.tag_name,
                 "tag_kind": r.tag_kind,
                 "min_confidence": r.min_confidence,
+                "applied_count": r.applied_count,
+                "coverage_count": r.coverage_count,
             }
             for r in rows
         ]
     )
 
 
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist/coverage", methods=["GET"])
+async def coverage(tag_id: int):
+    """Live "at threshold T, a sweep would cover ~N images" projection for the
+    allowlist tuning dashboard. Defaults to the tag's stored threshold."""
+    raw = request.args.get("threshold")
+    async with get_session() as session:
+        svc = AllowlistService(session)
+        if raw is not None:
+            try:
+                threshold = float(raw)
+            except ValueError:
+                return jsonify({"error": "threshold must be a float"}), 400
+            if not (0 < threshold <= 1):
+                return jsonify({"error": "threshold must be in (0, 1]"}), 400
+        else:
+            row = await session.get(TagAllowlist, tag_id)
+            if row is None:
+                return jsonify({"error": "not on allowlist"}), 404
+            threshold = row.min_confidence
+        count = await svc.coverage(tag_id, threshold)
+    return jsonify({"count": count, "threshold": threshold})
+
+
 @allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
 async def get_one(tag_id: int):
     async with get_session() as session:

backend/app/api/credentials.py

@@ -121,13 +121,15 @@ async def delete_credential(platform: str):
 
 @credentials_bp.route("/<platform>/verify", methods=["POST"])
 async def verify_credential(platform: str):
-    """Test the stored credential by running gallery-dl --simulate
-    against one of the platform's enabled sources. On success stamps
-    last_verified. Returns {valid: bool|null, reason, last_verified?}.
-    valid=null means "couldn't test" (no credential, or no enabled
-    source to point at)."""
+    """Test the stored credential against one of the platform's enabled sources,
+    WITHOUT downloading. Routes through the platform's backend
+    (download_backends.verify_credential) — native ingester for Patreon, an
+    authenticated API page; gallery-dl --simulate for the rest. On success
+    stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
+    valid=null means "couldn't test" (no credential, no enabled source, or an
+    inconclusive network/drift result)."""
     from ..models import Artist, Source
-    from ..services.gallery_dl import GalleryDLService, SourceConfig
+    from ..services.download_backends import verify_source_credential
 
     async with get_session() as session:
         if not await _ext_key_ok(session):
@@ -154,14 +156,14 @@ async def verify_credential(platform: str):
         cookies_path = await svc.get_cookies_path(platform)
         auth_token = await svc.get_token(platform)
 
-    gdl = GalleryDLService(images_root=Path("/images"))
-    ok, message = await gdl.verify(
+    ok, message = await verify_source_credential(
+        platform=platform,
         url=source.url,
         artist_slug=artist.slug,
-        platform=platform,
-        source_config=SourceConfig.from_dict(source.config_overrides or {}),
+        config_overrides=source.config_overrides or {},
         cookies_path=str(cookies_path) if cookies_path else None,
         auth_token=auth_token,
+        images_root=Path("/images"),
     )
 
     last_verified = None

backend/app/api/downloads.py

@@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
         "bytes_downloaded": event.bytes_downloaded,
         "error": event.error,
         "summary": _summary_from_metadata(event.metadata_),
+        # plan #709: mid-walk live counts for a RUNNING native-ingester event
+        # (None otherwise; phase 3 overwrites metadata with run_stats on finish).
+        "live": (event.metadata_ or {}).get("live"),
     }
 
 

backend/app/api/gallery.py

@@ -37,16 +37,30 @@ def _parse_filters():
     """Parse the composable gallery filters from query args, returning
     ``(filters_dict, sort)``. Raises ValueError (→ 400) on malformed ids/dates.
 
-    `tag_id` accepts a single id or a comma-separated list (AND); `media` is
-    image|video; `sort` is newest|oldest; `platform` selects one platform
-    (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are boolean
-    flags; `date_from`/`date_to` are inclusive calendar-day bounds (date_to is
-    widened by a day so the whole day is covered by the service's half-open
-    `< date_to`)."""
+    The structured tag filter (#6) is AND-of-OR plus exclusions:
+    - `tag_id` accepts a single id or a comma-separated list — all ANDed
+      (the include common case; back-compat).
+    - `tag_or` is REPEATABLE; each instance is a comma-separated OR-group, and
+      the image must match at least one tag from EACH group (groups ANDed).
+    - `tag_not` is a comma-separated exclude list (image must carry none).
+
+    `media` is image|video; `sort` is newest|oldest; `platform` selects one
+    platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are
+    boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds
+    (date_to is widened by a day so the whole day is covered by the service's
+    half-open `< date_to`)."""
     tag_raw = request.args.get("tag_id")
     tag_ids = (
         [int(x) for x in tag_raw.split(",") if x.strip()] if tag_raw else None
     ) or None
+    tag_or_groups = [
+        grp for raw in request.args.getlist("tag_or")
+        if (grp := [int(x) for x in raw.split(",") if x.strip()])
+    ] or None
+    not_raw = request.args.get("tag_not")
+    tag_exclude = (
+        [int(x) for x in not_raw.split(",") if x.strip()] if not_raw else None
+    ) or None
     post_id_raw = request.args.get("post_id")
     post_id = int(post_id_raw) if post_id_raw else None
     artist_id_raw = request.args.get("artist_id")
@@ -64,7 +78,9 @@ def _parse_filters():
         date_to += timedelta(days=1)  # inclusive of the date_to calendar day
     filters = {
         "tag_ids": tag_ids, "post_id": post_id, "artist_id": artist_id,
-        "media_type": media_type, "platform": platform,
+        "media_type": media_type,
+        "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
+        "platform": platform,
         "untagged": untagged, "no_artist": no_artist,
         "date_from": date_from, "date_to": date_to,
     }

backend/app/api/ml_admin.py

@@ -13,6 +13,10 @@ _EDITABLE = (
     "suggestion_threshold_general",
     "centroid_similarity_threshold",
     "min_reference_images",
+    "tagger_store_floor",
+    "video_frame_interval_seconds",
+    "video_max_frames",
+    "video_min_tag_frames",
 )
 
 
@@ -30,6 +34,10 @@ async def get_settings():
             "suggestion_threshold_general": s.suggestion_threshold_general,
             "centroid_similarity_threshold": s.centroid_similarity_threshold,
             "min_reference_images": s.min_reference_images,
+            "tagger_store_floor": s.tagger_store_floor,
+            "video_frame_interval_seconds": s.video_frame_interval_seconds,
+            "video_max_frames": s.video_max_frames,
+            "video_min_tag_frames": s.video_min_tag_frames,
             "tagger_model_version": s.tagger_model_version,
             "embedder_model_version": s.embedder_model_version,
         }
@@ -47,13 +55,54 @@ async def patch_settings():
         s = (
             await session.execute(select(MLSettings).where(MLSettings.id == 1))
         ).scalar_one()
+
+        # Merge the patch over current values, then validate the result as a
+        # whole — the store-floor invariant couples three fields, so they
+        # can't be checked one at a time.
+        proposed = {f: getattr(s, f) for f in _EDITABLE}
         for field in _EDITABLE:
             if field in body:
-                setattr(s, field, body[field])
+                proposed[field] = body[field]
+
+        err = _validate(proposed)
+        if err is not None:
+            return jsonify({"error": err}), 400
+
+        for field in _EDITABLE:
+            setattr(s, field, proposed[field])
         await session.commit()
     return await get_settings()
 
 
+def _validate(p: dict) -> str | None:
+    """Returns an error string if the proposed settings are invalid, else None.
+
+    Invariant (plan-task #764): the per-category suggestion thresholds can't
+    drop below tagger_store_floor — nothing below the floor is stored, so a
+    lower threshold would silently surface nothing in that gap. The UI clamps
+    the sliders to the floor; this is the server-side backstop.
+    """
+    floor = p["tagger_store_floor"]
+    if not (0.0 <= floor <= 1.0):
+        return "tagger_store_floor must be between 0 and 1"
+    for cat in ("character", "general"):
+        if p[f"suggestion_threshold_{cat}"] < floor:
+            return (
+                f"suggestion_threshold_{cat} cannot be below tagger_store_floor "
+                f"({floor}) — predictions below the floor are not stored"
+            )
+    # Video tagging (#747).
+    if p["video_frame_interval_seconds"] <= 0:
+        return "video_frame_interval_seconds must be > 0"
+    if p["video_max_frames"] < 1:
+        return "video_max_frames must be >= 1"
+    if p["video_min_tag_frames"] < 1:
+        return "video_min_tag_frames must be >= 1"
+    if p["video_min_tag_frames"] > p["video_max_frames"]:
+        return "video_min_tag_frames cannot exceed video_max_frames"
+    return None
+
+
 @ml_admin_bp.route("/backfill", methods=["POST"])
 async def trigger_backfill():
     from ..tasks.ml import backfill

backend/app/api/posts.py

@@ -17,6 +17,7 @@ async def list_posts():
     cursor = args.get("cursor") or None
     artist_id_raw = args.get("artist_id")
     platform = args.get("platform") or None
+    q = (args.get("q") or "").strip() or None
     limit_raw = args.get("limit", "24")
     direction = args.get("direction", "older")
     around_raw = args.get("around")
@@ -56,7 +57,7 @@ async def list_posts():
         if around_id is not None:
             result = await svc.around(
                 post_id=around_id, artist_id=artist_id,
-                platform=platform, limit=limit,
+                platform=platform, q=q, limit=limit,
             )
             if result is None:
                 return _bad("not_found", status=404, detail=f"post id={around_id}")
@@ -64,7 +65,7 @@ async def list_posts():
         try:
             page = await svc.scroll(
                 cursor=cursor, artist_id=artist_id,
-                platform=platform, limit=limit, direction=direction,
+                platform=platform, q=q, limit=limit, direction=direction,
             )
         except ValueError as exc:
             # Service raises ValueError for malformed cursors only;

backend/app/api/settings.py

@@ -25,6 +25,22 @@ _EDITABLE_FIELDS = (
     "download_schedule_default_seconds",
     "download_event_retention_days",
     "download_failure_warning_threshold",
+    "series_suggest_enabled",
+    "series_suggest_threshold",
+    "extdl_mega_enabled",
+    "extdl_gdrive_enabled",
+    "extdl_mediafire_enabled",
+    "extdl_dropbox_enabled",
+    "extdl_pixeldrain_enabled",
+)
+
+# Per-host external-download toggles — all plain booleans, validated uniformly.
+_EXTDL_TOGGLE_FIELDS = (
+    "extdl_mega_enabled",
+    "extdl_gdrive_enabled",
+    "extdl_mediafire_enabled",
+    "extdl_dropbox_enabled",
+    "extdl_pixeldrain_enabled",
 )
 
 
@@ -46,6 +62,13 @@ async def get_import_settings():
         "download_schedule_default_seconds": row.download_schedule_default_seconds,
         "download_event_retention_days": row.download_event_retention_days,
         "download_failure_warning_threshold": row.download_failure_warning_threshold,
+        "series_suggest_enabled": row.series_suggest_enabled,
+        "series_suggest_threshold": row.series_suggest_threshold,
+        "extdl_mega_enabled": row.extdl_mega_enabled,
+        "extdl_gdrive_enabled": row.extdl_gdrive_enabled,
+        "extdl_mediafire_enabled": row.extdl_mediafire_enabled,
+        "extdl_dropbox_enabled": row.extdl_dropbox_enabled,
+        "extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled,
     })
 
 
@@ -96,6 +119,22 @@ async def update_import_settings():
         if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
             return _bad_int("download_failure_warning_threshold", 1, 100)
 
+    if "series_suggest_enabled" in body and not isinstance(
+        body["series_suggest_enabled"], bool
+    ):
+        return jsonify(
+            {"error": "series_suggest_enabled must be a boolean"}
+        ), 400
+    for tog in _EXTDL_TOGGLE_FIELDS:
+        if tog in body and not isinstance(body[tog], bool):
+            return jsonify({"error": f"{tog} must be a boolean"}), 400
+    if "series_suggest_threshold" in body:
+        v = body["series_suggest_threshold"]
+        if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
+            return jsonify(
+                {"error": "series_suggest_threshold must be a number in [0, 1]"}
+            ), 400
+
     async with get_session() as session:
         row = await ImportSettings.load(session)
         for field in _EDITABLE_FIELDS:

backend/app/api/sources.py

@@ -85,6 +85,22 @@ async def create_source():
             return _bad("empty_url", detail=str(exc))
         except DuplicateSourceError as exc:
             return _bad("duplicate", status=409, existing_id=exc.existing_id)
+
+        # Immediate kickoff: a new enabled source is armed for backfill (#693)
+        # but would otherwise sit idle until the next scheduler tick (~60s).
+        # Enqueue the first walk now, skipping only if the platform is in a
+        # rate-limit cooldown (the scheduler picks it up when that clears).
+        dispatch_id = None
+        if record.enabled:
+            cooldowns = await active_platform_cooldowns(session)
+            if record.platform not in cooldowns:
+                session.add(DownloadEvent(source_id=record.id, status="pending"))
+                await session.commit()
+                dispatch_id = record.id
+
+    if dispatch_id is not None:
+        from ..tasks.download import download_source
+        download_source.delay(dispatch_id)
     return jsonify(record.to_dict()), 201
 
 
@@ -122,29 +138,123 @@ async def delete_source(source_id: int):
 
 @sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
 async def set_backfill(source_id: int):
-    """Plan #544: arm a source for backfill mode for the next N download
-    runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated
-    source dict. While backfill_runs_remaining > 0, downloads use
-    gallery-dl's full-walk config (skip: True + 30-min timeout) instead
-    of the catch-up default (skip: "exit:20" + 14.5-min timeout)."""
+    """Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
+    recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
+    (default "start"). 'start' walks the full post history in time-boxed chunks
+    until it reaches the bottom (then the source shows 'complete'); 'recover' is
+    the same walk but bypasses the Patreon seen-ledger to re-fetch
+    dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
+    re-grabs EVERY post's body + external links and localizes on-disk inline
+    images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
+    Returns the updated source dict (incl. backfill_state / backfill_chunks /
+    backfill_bypass_seen / backfill_recapture)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import (
+        uses_native_ingester,
+        verify_source_credential,
+    )
+    from .credentials import _get_crypto
+
     payload = await request.get_json(silent=True) or {}
-    runs = payload.get("runs", 3)
-    try:
-        runs = int(runs)
-    except (TypeError, ValueError):
-        return _bad("invalid_runs", detail="runs must be an integer")
+    action = payload.get("action", "start")
+    if action not in ("start", "stop", "recover", "recapture"):
+        return _bad(
+            "invalid_action",
+            detail="action must be 'start', 'stop', 'recover', or 'recapture'",
+        )
+
+    # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
+    # platform (where verify is one cheap API page), refuse if the credential is
+    # DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed
+    # on valid OR inconclusive (a network blip shouldn't block). Gated to native
+    # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
+    # an arm action. The credential read happens in a session that's CLOSED
+    # before the verify network call (don't hold a DB conn across the request).
+    if action in ("start", "recover", "recapture"):
+        async with get_session() as session:
+            rec = await SourceService(session).get(source_id)
+            if rec is None:
+                return _bad("not_found", status=404)
+            native = uses_native_ingester(rec.platform)
+            if native:
+                cred = CredentialService(session, _get_crypto())
+                cookies_path = await cred.get_cookies_path(rec.platform)
+                auth_token = await cred.get_token(rec.platform)
+        if native:
+            ok, message = await verify_source_credential(
+                platform=rec.platform,
+                url=rec.url,
+                artist_slug=rec.artist_slug,
+                config_overrides=rec.config_overrides or {},
+                cookies_path=str(cookies_path) if cookies_path else None,
+                auth_token=auth_token,
+                images_root=Path("/images"),
+            )
+            if ok is False:
+                return _bad("credential_rejected", detail=message, status=409)
+
     async with get_session() as session:
         try:
-            record = await SourceService(session).set_backfill_runs(
-                source_id, runs,
-            )
+            svc = SourceService(session)
+            if action == "start":
+                record = await svc.start_backfill(source_id)
+            elif action == "recover":
+                record = await svc.start_recovery(source_id)
+            elif action == "recapture":
+                record = await svc.start_recapture(source_id)
+            else:
+                record = await svc.stop_backfill(source_id)
         except LookupError:
             return _bad("not_found", status=404)
-        except ValueError as exc:
-            return _bad("invalid_runs", detail=str(exc))
     return jsonify(record.to_dict())
 
 
+@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
+async def preview_source_endpoint(source_id: int):
+    """Plan #708 B4: dry-run — count what a backfill WOULD download for a native
+    platform (Patreon today), without downloading. Walks the first few feed pages
+    and counts media not already in the seen/dead ledgers. Returns
+    {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
+    (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
+    cheap dry-run — their verify is a slow --simulate)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import preview_source, uses_native_ingester
+    from ..tasks._sync_engine import sync_session_factory
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        rec = await SourceService(session).get(source_id)
+        if rec is None:
+            return _bad("not_found", status=404)
+        if not uses_native_ingester(rec.platform):
+            return _bad(
+                "unsupported",
+                detail="Preview is only available for native-ingester platforms.",
+                status=400,
+            )
+        cred = CredentialService(session, _get_crypto())
+        cookies_path = await cred.get_cookies_path(rec.platform)
+
+    # The walk + ledger reads are sync (run off the request loop); the process
+    # sync engine is the same one the download task uses.
+    result = await preview_source(
+        platform=rec.platform,
+        url=rec.url,
+        source_id=source_id,
+        config_overrides=rec.config_overrides or {},
+        cookies_path=str(cookies_path) if cookies_path else None,
+        images_root=Path("/images"),
+        sync_session_factory=sync_session_factory(),
+    )
+    if "error" in result:
+        return _bad("preview_failed", detail=result["error"], status=409)
+    return jsonify(result)
+
+
 @sources_bp.route("/<int:source_id>/check", methods=["POST"])
 async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.

backend/app/api/suggestions.py

@@ -3,16 +3,48 @@
 from quart import Blueprint, jsonify, request
 
 from ..extensions import get_session
+from ..models import Tag, TagAllowlist
 from ..services.ml.allowlist import AllowlistService
 from ..services.ml.suggestions import SuggestionService
 
 suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
 
 
+async def _accept_payload(session, svc, newly_added: bool, tag_id: int) -> dict:
+    """Shape the accept/alias response. When accepting newly allowlists a tag,
+    include the coverage PROJECTION (at the tag's threshold) so the UI can show
+    a non-blocking "auto-applying to ~N images" toast — the actual apply runs
+    async via apply_allowlist_tags, so this is an estimate, not a post-hoc
+    count (#7)."""
+    payload = {"allowlisted": newly_added}
+    if newly_added:
+        tag = await session.get(Tag, tag_id)
+        row = await session.get(TagAllowlist, tag_id)
+        payload["tag_id"] = tag_id
+        payload["tag_name"] = tag.name if tag is not None else None
+        payload["projected_count"] = await svc.coverage(
+            tag_id, row.min_confidence if row is not None else 0.90,
+        )
+    return payload
+
+
 @suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
 async def get_suggestions(image_id: int):
+    # ?min=<float> overrides the configured per-category thresholds so the typed
+    # tag-input dropdown can surface EVERY stored prediction (min=0), including
+    # low-confidence actions/features, in canonical formatting. Omitted → the
+    # curated above-threshold list the Suggestions panel uses.
+    override = None
+    raw_min = request.args.get("min")
+    if raw_min is not None:
+        try:
+            override = min(1.0, max(0.0, float(raw_min)))
+        except ValueError:
+            return jsonify({"error": "min must be a float in [0,1]"}), 400
     async with get_session() as session:
-        sl = await SuggestionService(session).for_image(image_id)
+        sl = await SuggestionService(session).for_image(
+            image_id, threshold_override=override
+        )
     return jsonify(
         {
             "by_category": {
@@ -24,6 +56,11 @@ async def get_suggestions(image_id: int):
                         "score": round(s.score, 4),
                         "source": s.source,
                         "creates_new_tag": s.creates_new_tag,
+                        # raw model key (alias is stored under this) + whether an
+                        # operator alias produced this suggestion — drive the
+                        # modal's "Treat as alias"/"Remove alias" affordances.
+                        "raw_name": s.raw_name,
+                        "via_alias": s.via_alias,
                     }
                     for s in items
                 ]
@@ -42,13 +79,15 @@ async def accept_suggestion(image_id: int):
         return jsonify({"error": "tag_id required"}), 400
     tag_id = body["tag_id"]
     async with get_session() as session:
-        newly_added = await AllowlistService(session).accept(image_id, tag_id)
+        svc = AllowlistService(session)
+        newly_added = await svc.accept(image_id, tag_id)
+        payload = await _accept_payload(session, svc, newly_added, tag_id)
         await session.commit()
     if newly_added:
         from ..tasks.ml import apply_allowlist_tags
 
         apply_allowlist_tags.delay(tag_id=tag_id)
-    return "", 204
+    return jsonify(payload)
 
 
 @suggestions_bp.route(
@@ -59,19 +98,24 @@ async def alias_suggestion(image_id: int):
     required = {"alias_string", "alias_category", "canonical_tag_id"}
     if not body or not required.issubset(body):
         return jsonify({"error": f"required: {sorted(required)}"}), 400
+    canonical_tag_id = body["canonical_tag_id"]
     async with get_session() as session:
-        newly_added = await AllowlistService(session).add_alias_and_accept(
+        svc = AllowlistService(session)
+        newly_added = await svc.add_alias_and_accept(
             image_id,
             body["alias_string"],
             body["alias_category"],
-            body["canonical_tag_id"],
+            canonical_tag_id,
+        )
+        payload = await _accept_payload(
+            session, svc, newly_added, canonical_tag_id,
         )
         await session.commit()
     if newly_added:
         from ..tasks.ml import apply_allowlist_tags
 
-        apply_allowlist_tags.delay(tag_id=body["canonical_tag_id"])
-    return "", 204
+        apply_allowlist_tags.delay(tag_id=canonical_tag_id)
+    return jsonify(payload)
 
 
 @suggestions_bp.route(

backend/app/api/system_activity.py

@@ -31,7 +31,7 @@ system_activity_bp = Blueprint(
 # absent.
 _QUEUE_NAMES = (
     "default", "import", "thumbnail", "ml",
-    "download", "scan", "maintenance",
+    "download", "scan", "maintenance", "maintenance_long",
 )
 
 # Cache module-level so all requests share the cache between polls.
@@ -147,6 +147,7 @@ async def list_runs():
     """Paginated task_run history. Query params:
       queue=<name>     filter to one queue
       status=<status>  filter to one status (running/ok/error/timeout/retry)
+      task=<substr>    case-insensitive substring match on task_name
       limit=<int>      default 50, max 200
       before_id=<int>  cursor for keyset pagination
 
@@ -161,6 +162,7 @@ async def list_runs():
 
     queue = request.args.get("queue")
     status = request.args.get("status")
+    task = request.args.get("task")
     before_id_raw = request.args.get("before_id")
     before_id = int(before_id_raw) if before_id_raw else None
 
@@ -170,6 +172,11 @@ async def list_runs():
             stmt = stmt.where(TaskRun.queue == queue)
         if status:
             stmt = stmt.where(TaskRun.status == status)
+        if task:
+            # Task names contain literal underscores (download_source,
+            # vacuum_analyze) — escape LIKE wildcards so a search for
+            # "vacuum_analyze" doesn't treat "_" as a single-char match.
+            stmt = stmt.where(TaskRun.task_name.ilike(f"%{_escape_like(task)}%", escape="\\"))
         if before_id is not None:
             stmt = stmt.where(TaskRun.id < before_id)
         stmt = stmt.limit(limit + 1)
@@ -225,6 +232,12 @@ async def list_failures():
     })
 
 
+def _escape_like(value: str) -> str:
+    """Escape SQL LIKE/ILIKE metacharacters so user search text is matched
+    literally. Pairs with `escape="\\"` on the .ilike() call."""
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
 def _row_to_dict(r: TaskRun) -> dict:
     return {
         "id": r.id,

backend/app/api/tags.py

@@ -8,12 +8,16 @@ from ..extensions import get_session
 from ..models import Tag, TagKind
 from ..models.tag_allowlist import TagAllowlist
 from ..services.bulk_tag_service import BulkTagService
+from ..services.ml.aliases import AliasService
+from ..services.series_match_service import SeriesMatchService
 from ..services.series_service import SeriesError, SeriesService
 from ..services.tag_directory_service import TagDirectoryService
+from ..services.tag_query import serialize_tag
 from ..services.tag_service import (
     TagMergeConflict,
     TagService,
     TagValidationError,
+    normalize_tag_name,
 )
 from ..utils.tag_prefix import parse_kind_prefix
 
@@ -70,17 +74,7 @@ async def autocomplete():
         hits = await svc.autocomplete(q, kind=kind, limit=limit)
 
     return jsonify(
-        [
-            {
-                "id": h.id,
-                "name": h.name,
-                "kind": h.kind,
-                "fandom_id": h.fandom_id,
-                "fandom_name": h.fandom_name,
-                "image_count": h.image_count,
-            }
-            for h in hits
-        ]
+        [{**serialize_tag(h), "image_count": h.image_count} for h in hits]
     )
 
 
@@ -141,6 +135,11 @@ async def create_tag():
 
     fandom_id = body.get("fandom_id")
 
+    # #701: Title-Case operator-entered tags. Only here (the explicit create
+    # endpoint), NOT in the shared find_or_create — the ML tagger uses that path
+    # and must keep the booru vocabulary's casing for allowlist matching.
+    name = normalize_tag_name(name)
+
     async with get_session() as session:
         svc = TagService(session)
         try:
@@ -158,17 +157,7 @@ async def list_tags_for_image(image_id: int):
     async with get_session() as session:
         svc = TagService(session)
         tags = await svc.list_for_image(image_id)
-    return jsonify(
-        [
-            {
-                "id": t.id,
-                "name": t.name,
-                "kind": t.kind.value,
-                "fandom_id": t.fandom_id,
-            }
-            for t in tags
-        ]
-    )
+    return jsonify([serialize_tag(t) for t in tags])
 
 
 @tags_bp.route("/images/<int:image_id>/tags", methods=["POST"])
@@ -212,6 +201,25 @@ async def get_tag(tag_id: int):
         )
 
 
+@tags_bp.route("/tags/<int:tag_id>/aliases", methods=["GET"])
+async def list_tag_aliases(tag_id: int):
+    """Model keys that fold into this tag (tag-side alias view). Remove via the
+    shared DELETE /api/aliases/<string>/<category>."""
+    async with get_session() as session:
+        if await session.get(Tag, tag_id) is None:
+            return jsonify({"error": "tag not found"}), 404
+        rows = await AliasService(session).list_for_tag(tag_id)
+    return jsonify(
+        [
+            {
+                "alias_string": r.alias_string,
+                "alias_category": r.alias_category,
+            }
+            for r in rows
+        ]
+    )
+
+
 @tags_bp.route("/tags/<int:tag_id>", methods=["PATCH"])
 async def update_tag(tag_id: int):
     """Rename and/or re-fandom a tag. Body may carry `name` and/or
@@ -312,6 +320,27 @@ async def common_tags():
     return jsonify({"tags": tags})
 
 
+@tags_bp.route("/images/cluster/tag-gaps", methods=["POST"])
+async def cluster_tag_gaps():
+    """Cluster-consensus tag gaps for a visual neighbour set (#94 Explore):
+    tags on >= threshold of the images but not all. Each gap carries the
+    laggard image ids (minus rejections) for apply-to-cluster."""
+    body = await request.get_json()
+    ids, err = _parse_bulk_ids(body)
+    if err:
+        return err
+    try:
+        threshold = float(body.get("threshold", 0.6))
+    except (TypeError, ValueError):
+        threshold = 0.6
+    threshold = min(1.0, max(0.0, threshold))
+    async with get_session() as session:
+        gaps = await BulkTagService(session).tag_gaps(ids, threshold)
+    return jsonify(
+        {"gaps": gaps, "total": len(set(ids)), "threshold": threshold}
+    )
+
+
 @tags_bp.route("/images/bulk/tags", methods=["POST"])
 async def bulk_add_tag():
     body = await request.get_json()
@@ -362,6 +391,31 @@ def _series_err(exc: SeriesError):
     return jsonify({"error": msg}), status
 
 
+def _opt_int(body, key: str):
+    """(value, error) — value is None when absent, error is (json, status)."""
+    if not body or body.get(key) is None:
+        return None, None
+    try:
+        return int(body[key]), None
+    except (TypeError, ValueError):
+        return None, (jsonify({"error": f"{key} must be an integer"}), 400)
+
+
+def _parse_int_list(body, key: str, *, max_ids: int = 500):
+    """(list, error) for a required list of ints under `key`."""
+    if not body or key not in body:
+        return None, (jsonify({"error": f"{key} required"}), 400)
+    raw = body[key]
+    if not isinstance(raw, list) or not raw:
+        return None, (jsonify({"error": f"{key} must be a non-empty list"}), 400)
+    if len(raw) > max_ids:
+        return None, (jsonify({"error": f"too many ids (max {max_ids})"}), 400)
+    try:
+        return [int(x) for x in raw], None
+    except (TypeError, ValueError):
+        return None, (jsonify({"error": f"{key} must be integers"}), 400)
+
+
 @tags_bp.route("/series/<int:tag_id>/pages", methods=["GET"])
 async def series_pages(tag_id: int):
     async with get_session() as session:
@@ -402,15 +456,26 @@ async def series_remove(tag_id: int):
     return jsonify({"removed_count": n})
 
 
-@tags_bp.route("/series/<int:tag_id>/reorder", methods=["POST"])
-async def series_reorder(tag_id: int):
-    body = await request.get_json()
-    ids, err = _parse_bulk_ids(body, max_ids=500)
-    if err:
-        return err
+@tags_bp.route("/series/<int:tag_id>/pages/number", methods=["POST"])
+async def series_set_page_number(tag_id: int):
+    """Set one placed page's number — the operator's value (sparse, gaps
+    allowed); pass page_number: null to leave it unnumbered."""
+    body = await request.get_json() or {}
+    image_id, ierr = _opt_int(body, "image_id")
+    if ierr:
+        return ierr
+    if image_id is None:
+        return jsonify({"error": "image_id required"}), 400
+    if "page_number" not in body:
+        return jsonify({"error": "page_number required (may be null)"}), 400
+    page_number, perr = _opt_int(body, "page_number")
+    if perr:
+        return perr
     async with get_session() as session:
         try:
-            await SeriesService(session).reorder(tag_id, ids)
+            await SeriesService(session).set_page_number(
+                tag_id, image_id, page_number
+            )
         except SeriesError as exc:
             return _series_err(exc)
         await session.commit()
@@ -433,3 +498,201 @@ async def series_cover(tag_id: int):
             return _series_err(exc)
         await session.commit()
     return jsonify({"ok": True})
+
+
+# ---- chapter dividers (FC-6.x) -------------------------------------------
+# A chapter is a cosmetic divider anchored to the page that begins it; it owns
+# no pages. Page ordering follows each page's operator-set number (the
+# /pages/number endpoint), so there is no per-chapter reorder/merge — those are
+# gone.
+
+
+@tags_bp.route("/series/<int:tag_id>/chapters", methods=["POST"])
+async def series_chapter_create(tag_id: int):
+    body = await request.get_json() or {}
+    anchor, aerr = _opt_int(body, "anchor_image_id")
+    if aerr:
+        return aerr
+    if anchor is None:
+        return jsonify({"error": "anchor_image_id required"}), 400
+    title = body.get("title")
+    if title is not None and not isinstance(title, str):
+        return jsonify({"error": "title must be a string"}), 400
+    part, perr = _opt_int(body, "stated_part")
+    if perr:
+        return perr
+    async with get_session() as session:
+        try:
+            ch = await SeriesService(session).create_divider(
+                tag_id, anchor, title=title, stated_part=part,
+            )
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify(ch)
+
+
+@tags_bp.route(
+    "/series/<int:tag_id>/chapters/<int:chapter_id>", methods=["PATCH"]
+)
+async def series_chapter_update(tag_id: int, chapter_id: int):
+    body = await request.get_json() or {}
+    kwargs: dict = {}
+    if "title" in body:
+        if body["title"] is not None and not isinstance(body["title"], str):
+            return jsonify({"error": "title must be a string"}), 400
+        kwargs.update(set_title=True, title=body["title"])
+    if "stated_part" in body:
+        part, perr = _opt_int(body, "stated_part")
+        if perr:
+            return perr
+        kwargs.update(set_part=True, stated_part=part)
+    if "anchor_image_id" in body:
+        anchor, aerr = _opt_int(body, "anchor_image_id")
+        if aerr:
+            return aerr
+        if anchor is None:
+            return jsonify(
+                {"error": "anchor_image_id must be an integer"}
+            ), 400
+        kwargs.update(set_anchor=True, anchor_image_id=anchor)
+    async with get_session() as session:
+        try:
+            await SeriesService(session).update_divider(
+                tag_id, chapter_id, **kwargs
+            )
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify({"ok": True})
+
+
+@tags_bp.route(
+    "/series/<int:tag_id>/chapters/<int:chapter_id>", methods=["DELETE"]
+)
+async def series_chapter_delete(tag_id: int, chapter_id: int):
+    async with get_session() as session:
+        try:
+            await SeriesService(session).delete_divider(tag_id, chapter_id)
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify({"ok": True})
+
+
+# ---- browse list + post→series flows (FC-6.2) -----------------------------
+
+
+@tags_bp.route("/series", methods=["GET"])
+async def series_list():
+    args = request.args
+    sort = args.get("sort", "recent")
+    if sort not in ("recent", "name", "size"):
+        return jsonify({"error": "sort must be recent|name|size"}), 400
+    artist_id = None
+    if args.get("artist_id") is not None:
+        try:
+            artist_id = int(args["artist_id"])
+        except ValueError:
+            return jsonify({"error": "artist_id must be an integer"}), 400
+    async with get_session() as session:
+        rows = await SeriesService(session).list_series(
+            sort=sort, artist_id=artist_id
+        )
+    return jsonify({"series": rows})
+
+
+@tags_bp.route("/series/from-post", methods=["POST"])
+async def series_from_post():
+    body = await request.get_json()
+    post_id, err = _opt_int(body, "post_id")
+    if err:
+        return err
+    if post_id is None:
+        return jsonify({"error": "post_id required"}), 400
+    async with get_session() as session:
+        try:
+            out = await SeriesService(session).promote_post_to_series(post_id)
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify(out)
+
+
+@tags_bp.route("/series/<int:tag_id>/add-post", methods=["POST"])
+async def series_add_post(tag_id: int):
+    body = await request.get_json()
+    post_id, err = _opt_int(body, "post_id")
+    if err:
+        return err
+    if post_id is None:
+        return jsonify({"error": "post_id required"}), 400
+    async with get_session() as session:
+        try:
+            out = await SeriesService(session).add_post(tag_id, post_id)
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify(out)
+
+
+@tags_bp.route("/series/<int:tag_id>/pending/place", methods=["POST"])
+async def series_place_pending(tag_id: int):
+    """Place staged (pending) pages into the run, numbered sequentially from
+    `start_page` in the given order (#789). start_page null → unnumbered."""
+    body = await request.get_json()
+    ids, err = _parse_bulk_ids(body, max_ids=500)
+    if err:
+        return err
+    start, serr = _opt_int(body, "start_page")
+    if serr:
+        return serr
+    async with get_session() as session:
+        try:
+            n = await SeriesService(session).place_pending(
+                tag_id, ids, start_page=start
+            )
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify({"placed_count": n})
+
+
+# ---- suggestion queue (FC-6.3) --------------------------------------------
+
+
+@tags_bp.route("/series/suggestions", methods=["GET"])
+async def series_suggestions_list():
+    async with get_session() as session:
+        rows = await SeriesMatchService(session).list_pending()
+    return jsonify({"suggestions": rows})
+
+
+@tags_bp.route("/series/suggestions/<int:sid>/accept", methods=["POST"])
+async def series_suggestion_accept(sid: int):
+    async with get_session() as session:
+        try:
+            out = await SeriesMatchService(session).accept(sid)
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify(out)
+
+
+@tags_bp.route("/series/suggestions/<int:sid>/dismiss", methods=["POST"])
+async def series_suggestion_dismiss(sid: int):
+    async with get_session() as session:
+        try:
+            await SeriesMatchService(session).dismiss(sid)
+        except SeriesError as exc:
+            return _series_err(exc)
+        await session.commit()
+    return jsonify({"ok": True})
+
+
+@tags_bp.route("/series/suggestions/rescan", methods=["POST"])
+async def series_suggestions_rescan():
+    from ..tasks.admin import rescan_series_suggestions_task
+
+    res = rescan_series_suggestions_task.delay()
+    return jsonify({"task_id": res.id})

backend/app/celery_app.py

@@ -30,6 +30,7 @@ def make_celery() -> Celery:
             "backend.app.tasks.maintenance",
             "backend.app.tasks.ml",
             "backend.app.tasks.download",
+            "backend.app.tasks.external",
             "backend.app.tasks.backup",
             "backend.app.tasks.admin",
             "backend.app.tasks.library_audit",
@@ -42,16 +43,51 @@ def make_celery() -> Celery:
             "backend.app.tasks.ml.*": {"queue": "ml"},
             "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
             "backend.app.tasks.download.*": {"queue": "download"},
+            # External file-host fetches are downloads — same lane (they can run
+            # long, but the download worker already tolerates long backfills).
+            "backend.app.tasks.external.*": {"queue": "download"},
             "backend.app.tasks.scan.*": {"queue": "scan"},
+            # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
+            # (concurrency-1 on the scheduler). The long one-shots (DB backups,
+            # library audits, admin maintenance: normalize/re-extract/cascade-
+            # delete) run on a SEPARATE `maintenance_long` lane + worker so they
+            # can never starve the quick self-healing sweeps (operator-flagged
+            # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
             "backend.app.tasks.maintenance.*": {"queue": "maintenance"},
-            "backend.app.tasks.backup.*": {"queue": "maintenance"},
-            "backend.app.tasks.admin.*": {"queue": "maintenance"},
-            "backend.app.tasks.library_audit.*": {"queue": "maintenance"},
+            "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
+            "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
+            "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
         },
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,
         worker_prefetch_multiplier=1,
+        # Broker resilience (2026-06-24): a swarm overlay-network blip after a
+        # redeploy left Redis healthy but transiently unreachable, and a worker
+        # starting in that window crash-looped on the initial broker connect
+        # (kombu OperationalError) instead of waiting it out — needing a manual
+        # Redis reset to recover. Retry the broker FOREVER (None) on startup and
+        # at runtime so a transient outage self-heals when routing returns,
+        # rather than the worker exiting.
         broker_connection_retry_on_startup=True,
+        broker_connection_retry=True,
+        broker_connection_max_retries=None,
+        # Redis-transport socket options (apply to the BROKER connection): a
+        # short connect timeout + TCP keepalive so a dead/blocked socket is
+        # noticed and retried, and a periodic health check that proactively
+        # reconnects a live worker through a network hiccup.
+        broker_transport_options={
+            "socket_connect_timeout": 5,
+            "socket_timeout": 30,
+            "socket_keepalive": True,
+            "retry_on_timeout": True,
+            "health_check_interval": 30,
+        },
+        # Same hardening for the Redis RESULT backend (separate connection pool).
+        redis_socket_connect_timeout=5,
+        redis_socket_timeout=30,
+        redis_socket_keepalive=True,
+        redis_retry_on_timeout=True,
+        redis_backend_health_check_interval=30,
         beat_schedule={
             "recover-interrupted-tasks": {
                 "task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
@@ -144,6 +180,21 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.thumbnail.backfill_thumbnails",
                 "schedule": 86400.0,
             },
+            # External file-host downloads (#830): a steady sweep catches links
+            # the post-download hook missed (worker down, etc.); recovery re-tries
+            # dead links daily; retention prunes long-dead rows.
+            "extdl-sweep": {
+                "task": "backend.app.tasks.external.sweep_external_links",
+                "schedule": 600.0,  # every 10 min
+            },
+            "extdl-recover-daily": {
+                "task": "backend.app.tasks.external.recover_external_links",
+                "schedule": 86400.0,
+            },
+            "extdl-prune-daily": {
+                "task": "backend.app.tasks.external.prune_external_links",
+                "schedule": 86400.0,
+            },
         },
         timezone="UTC",
     )

backend/app/celery_signals.py

@@ -69,7 +69,15 @@ def _queue_for(task) -> str:
         return "ml"
     if name.startswith("backend.app.tasks.thumbnail."):
         return "thumbnail"
-    if name.startswith("backend.app.tasks.download."):
+    if name.startswith((
+        "backend.app.tasks.download.",
+        # External file-host fetches share the download lane (celery_app
+        # routes external.* → download). Mirror it here or TaskRun.queue
+        # lies 'default' for them, so per-queue dashboard filters and the
+        # per-queue threshold override miss them — the same gap the
+        # 2026-06-02 audit fixed for backup/admin/library_audit.
+        "backend.app.tasks.external.",
+    )):
         return "download"
     if name.startswith("backend.app.tasks.scan."):
         return "scan"

backend/app/models/__init__.py

@@ -7,6 +7,8 @@ from .backup_run import BackupRun
 from .base import Base
 from .credential import Credential
 from .download_event import DownloadEvent
+from .external_link import ExternalLink
+from .image_prediction import ImagePrediction
 from .image_provenance import ImageProvenance
 from .image_record import ImageRecord
 from .import_batch import ImportBatch
@@ -14,10 +16,16 @@ from .import_settings import ImportSettings
 from .import_task import ImportTask
 from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
+from .patreon_failed_media import PatreonFailedMedia
+from .patreon_seen_media import PatreonSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
+from .series_chapter import SeriesChapter
 from .series_page import SeriesPage
+from .series_suggestion import SeriesSuggestion
 from .source import Source
+from .subscribestar_failed_media import SubscribeStarFailedMedia
+from .subscribestar_seen_media import SubscribeStarSeenMedia
 from .tag import Tag, TagKind, image_tag
 from .tag_alias import TagAlias
 from .tag_allowlist import TagAllowlist
@@ -33,15 +41,23 @@ __all__ = [
     "BackupRun",
     "Source",
     "Credential",
+    "PatreonFailedMedia",
+    "PatreonSeenMedia",
+    "SubscribeStarFailedMedia",
+    "SubscribeStarSeenMedia",
     "Post",
     "PostAttachment",
+    "SeriesChapter",
     "SeriesPage",
+    "SeriesSuggestion",
     "ImageRecord",
+    "ImagePrediction",
     "ImageProvenance",
     "Tag",
     "TagKind",
     "image_tag",
     "DownloadEvent",
+    "ExternalLink",
     "ImportBatch",
     "ImportTask",
     "ImportSettings",

backend/app/models/external_link.py

@@ -0,0 +1,73 @@
+"""ExternalLink — an off-platform file-host link found in a post body.
+
+Creators host the actual files (films, packs) on mega.nz / Google Drive /
+MediaFire / Dropbox / Pixeldrain and drop the link in the post text. This row
+is the record that the link existed (so nothing is silently dropped), the
+dedup + dead-letter ledger for fetching it, and the driver the download worker
+walks. `url` keeps the FULL link including the `#fragment` (mega's decryption
+key) — truncating it makes the file undownloadable.
+
+status lifecycle: pending → downloading → downloaded | failed | dead
+(too many attempts) | skipped (host disabled). `attachment_id` links the
+captured file once a download lands (SET NULL so deleting the attachment
+doesn't delete the link record).
+"""
+
+from datetime import datetime
+
+from sqlalchemy import (
+    DateTime,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    func,
+    text,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+# Kept in sync with link_extract.SUPPORTED_HOSTS and the CHECK in migration 0049.
+HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
+STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
+
+
+class ExternalLink(Base):
+    __tablename__ = "external_link"
+    __table_args__ = (
+        # One row per (post, url). The full url (incl. #fragment) is the identity
+        # — the same file linked twice in a post collapses to one row.
+        Index("uq_external_link_post_url", "post_id", "url", unique=True),
+        Index("ix_external_link_status", "status"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    post_id: Mapped[int] = mapped_column(
+        ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    artist_id: Mapped[int | None] = mapped_column(
+        ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
+    )
+    host: Mapped[str] = mapped_column(String(16), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    label: Mapped[str | None] = mapped_column(Text, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, server_default="pending"
+    )
+    attempts: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    attachment_id: Mapped[int | None] = mapped_column(
+        ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    completed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)

backend/app/models/image_prediction.py

@@ -0,0 +1,37 @@
+"""ImagePrediction — one row per (image, tagger vocab prediction).
+
+Replaces the image_record.tagger_predictions JSON blob (#768). Storing the
+raw Camie/booru vocab name (not a tag_id) preserves the suggestion read
+path's semantics: raw_name → canonical Tag resolution happens at read time
+via the alias map, and accepting a prediction can CREATE the Tag. The store
+floor (ml_settings.tagger_store_floor) is applied at WRITE time, so only
+predictions >= the floor land here.
+"""
+
+from sqlalchemy import Float, ForeignKey, Index, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class ImagePrediction(Base):
+    __tablename__ = "image_prediction"
+    __table_args__ = (
+        UniqueConstraint(
+            "image_record_id", "raw_name", name="image_raw_name",
+        ),
+        # Per-image read (suggestion build) and the "images with tag X above
+        # Y" query the JSON blob never allowed.
+        Index("ix_image_prediction_image", "image_record_id"),
+        Index("ix_image_prediction_name_score", "raw_name", "score"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    image_record_id: Mapped[int] = mapped_column(
+        ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False,
+    )
+    # The raw tagger vocab key (booru form) — NOT a tag_id. Resolved to a
+    # canonical Tag at read time, exactly as the old JSON keys were.
+    raw_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    category: Mapped[str] = mapped_column(String(64), nullable=False)
+    score: Mapped[float] = mapped_column(Float, nullable=False)

backend/app/models/image_provenance.py

@@ -41,6 +41,16 @@ class ImageProvenance(Base):
     source_id: Mapped[int | None] = mapped_column(
         ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True
     )
+    # The archive PostAttachment this image was extracted FROM, when it came
+    # out of a .zip/.rar rather than as a loose file (milestone #87). Lets the
+    # provenance UI show the exact archive a file lives inside instead of every
+    # attachment on the post. NULL for loose downloads and pre-backfill rows.
+    # SET NULL so deleting the archive attachment never destroys the (image,
+    # post) edge — it just forgets which archive it came from.
+    from_attachment_id: Mapped[int | None] = mapped_column(
+        ForeignKey("post_attachment.id", ondelete="SET NULL"),
+        nullable=True, index=True,
+    )
     captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
     captured_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()

backend/app/models/image_record.py

@@ -13,6 +13,7 @@ from sqlalchemy import (
     BigInteger,
     DateTime,
     Enum,
+    Float,
     ForeignKey,
     Integer,
     String,
@@ -39,6 +40,10 @@ class ImageRecord(Base):
     mime: Mapped[str] = mapped_column(String(64), nullable=False)
     width: Mapped[int | None] = mapped_column(Integer, nullable=True)
     height: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    # Video container duration (seconds); NULL for images. The Tier-1 video
+    # near-dup key (#871): two videos of the same artist with matching duration
+    # (+ aspect) are the same content across re-encodes — dedup like image pHash.
+    duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
 
     # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
     # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
@@ -49,6 +54,18 @@ class ImageRecord(Base):
     # Thumbnail (populated by FC-2)
     thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
 
+    # Source provenance for downloaded media (#830 Phase 2). `source_url` is the
+    # CDN/origin URL the file was fetched from (debugging + future re-fetch).
+    # `source_filehash` is the URL's 32-hex CDN identity segment
+    # (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's
+    # inline `<img src=CDN>` back to this local copy so the rendered body serves
+    # our stored image instead of hotlinking the public source. Indexed for the
+    # render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows.
+    source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+    source_filehash: Mapped[str | None] = mapped_column(
+        String(32), nullable=True, index=True
+    )
+
     # Origin / provenance pointers
     origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
     primary_post_id: Mapped[int | None] = mapped_column(
@@ -60,8 +77,10 @@ class ImageRecord(Base):
         ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
     )
 
-    # ML fields (populated by FC-2's ml-worker)
-    tagger_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    # ML fields (populated by FC-2's ml-worker). Per-tag predictions live in the
+    # normalized image_prediction table (#768) — the tagger_predictions JSON
+    # column was dropped in migration 0046. tagger_model_version stays as the
+    # "has this been tagged / is it current?" signal the backfill sweep reads.
     tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
     # 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
     # a column-width migration.

backend/app/models/import_settings.py

@@ -64,6 +64,34 @@ class ImportSettings(Base):
         Integer, nullable=False, default=3,
     )
 
+    # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
+    # the weighted-score cut-off (0..1) above which a pending suggestion is made.
+    series_suggest_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True,
+    )
+    series_suggest_threshold: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.5,
+    )
+
+    # #830 off-platform file-host downloads — per-host enable lever (default on,
+    # rule #26). Column names are extdl_<host>_enabled so the worker reads them
+    # via getattr(settings, f"extdl_{host}_enabled", True).
+    extdl_mega_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default="true",
+    )
+    extdl_gdrive_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default="true",
+    )
+    extdl_mediafire_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default="true",
+    )
+    extdl_dropbox_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default="true",
+    )
+    extdl_pixeldrain_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default="true",
+    )
+
     @classmethod
     async def load(cls, session) -> ImportSettings:
         """The singleton settings row (id=1), via an async session."""

backend/app/models/library_audit_run.py

@@ -35,3 +35,10 @@ class LibraryAuditRun(Base):
     matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes
+    # from, and the last time a chunk made progress (so the recovery sweep can
+    # tell a progressing multi-chunk audit from a stuck one).
+    resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    last_progress_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )

backend/app/models/ml_settings.py

@@ -28,9 +28,33 @@ class MLSettings(Base):
     centroid_similarity_threshold: Mapped[float] = mapped_column(
         Float, nullable=False, default=0.55
     )
+    # Ingest floor: tagger predictions below this confidence are not stored
+    # (tagger.Tagger.infer). Default 0.70 — the suggestion path already
+    # filters at 0.70 and the centroid/learned path covers low-confidence
+    # preferred tags, so the sub-0.70 tail is redundant weight (it had
+    # bloated image_record's TOAST to ~100 GB; plan-task #764). Operator-
+    # tunable via Settings → ML; must stay ≤ the suggestion thresholds.
+    tagger_store_floor: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.70
+    )
     min_reference_images: Mapped[int] = mapped_column(
         Integer, nullable=False, default=5
     )
+    # Video tagging (#747). Sample one frame every N seconds (fixed CADENCE, not a
+    # fixed count) so a tag's frame-presence reflects real screen time regardless
+    # of video length; cap the total so a long video can't explode into hundreds
+    # of inferences (the cadence stretches past the cap). A tag is kept only if it
+    # appears in >= video_min_tag_frames sampled frames (≈ that many × interval
+    # seconds on screen) — duration-independent noise rejection. Operator-tunable.
+    video_frame_interval_seconds: Mapped[float] = mapped_column(
+        Float, nullable=False, default=4.0
+    )
+    video_max_frames: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=64
+    )
+    video_min_tag_frames: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=3
+    )
     tagger_model_version: Mapped[str] = mapped_column(
         String(128), nullable=False, default="camie-tagger-v2"
     )

backend/app/models/patreon_failed_media.py

@@ -0,0 +1,45 @@
+"""PatreonFailedMedia — per-source dead-letter ledger of Patreon media that
+keeps failing to download/validate.
+
+Plan #705 (#7). A media that fails every walk (404'd CDN URL, deleted post,
+geo-blocked Mux stream, persistently-corrupt bytes) would otherwise re-error
+forever and re-burn backfill chunks. After ``attempts`` reaches the dead-letter
+threshold the ingester skips it on routine tick/backfill walks (recovery still
+re-attempts it — the operator's "try everything again"). A later clean download
+clears the row (the media recovered).
+
+`filehash` is the same per-media key the seen-ledger uses (32-hex CDN MD5, or a
+``video:`` / ``post:filename`` synthesized key) — hence String(128). UNIQUE
+(source_id, filehash) is the upsert key.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PatreonFailedMedia(Base):
+    __tablename__ = "patreon_failed_media"
+    __table_args__ = (
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_failed_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    first_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/patreon_seen_media.py

@@ -0,0 +1,38 @@
+"""PatreonSeenMedia — per-source ledger of Patreon media already
+downloaded+processed.
+
+Replaces gallery-dl's archive.sqlite3 with our own queryable table so
+routine walks can skip media we've already ingested (and a future
+"recovery" mode can deliberately bypass the ledger to re-walk).
+
+`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos —
+which have no stable content hash at discovery time — use a sentinel of
+the form ``video:<post_id>:<media_id>``, hence String(128) rather than 32.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PatreonSeenMedia(Base):
+    __tablename__ = "patreon_seen_media"
+    __table_args__ = (
+        # Dedup key the downloader upserts against: one ledger row per
+        # (source, media). A second sighting of the same media is a no-op.
+        UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    seen_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/post_attachment.py

@@ -14,10 +14,12 @@ from sqlalchemy import (
     BigInteger,
     DateTime,
     ForeignKey,
+    Index,
     Integer,
     String,
     Text,
     func,
+    text,
 )
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -26,6 +28,24 @@ from .base import Base
 
 class PostAttachment(Base):
     __tablename__ = "post_attachment"
+    # Dedup is PER-POST, not global (2026-06-08): the same non-art file attached
+    # to many posts gets one row per post over a single sha-addressed blob, so no
+    # post is left a bare shell. Partial uniques: (post_id, sha256) for real posts;
+    # (sha256) alone for the NULL-post filesystem case (one row per file there).
+    __table_args__ = (
+        Index(
+            "uq_post_attachment_post_sha",
+            "post_id", "sha256",
+            unique=True,
+            postgresql_where=text("post_id IS NOT NULL"),
+        ),
+        Index(
+            "uq_post_attachment_null_post_sha",
+            "sha256",
+            unique=True,
+            postgresql_where=text("post_id IS NULL"),
+        ),
+    )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     post_id: Mapped[int | None] = mapped_column(
@@ -35,7 +55,7 @@ class PostAttachment(Base):
         ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
     )
     sha256: Mapped[str] = mapped_column(
-        String(64), nullable=False, unique=True, index=True
+        String(64), nullable=False, index=True
     )
     path: Mapped[str] = mapped_column(Text, nullable=False)
     original_filename: Mapped[str] = mapped_column(Text, nullable=False)

backend/app/models/series_chapter.py

@@ -0,0 +1,47 @@
+"""SeriesChapter — a cosmetic chapter DIVIDER within a series (FC-6.x reframe).
+
+A series is ONE flat, series-global ordered run of SeriesPages. A chapter is NOT
+a container — it owns no pages. It is a labeled divider anchored to the page that
+BEGINS the chapter (anchor_page_id → series_page): "a new chapter starts here."
+A page's chapter is derived at read time as the nearest preceding divider.
+
+Dividers never affect page ordering or the series-global page numbers; they stay
+pinned to their anchor page across reorders. anchor_page_id is UNIQUE — at most
+one chapter begins at a given page — and FK-cascades, so removing the anchor page
+from the series drops the divider (the chapter merges into the preceding run).
+
+title is the optional chapter name; stated_part is the optional operator-facing
+"Part N" label (shown instead of a derived ordinal when set).
+"""
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, ForeignKey, Integer, Text, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class SeriesChapter(Base):
+    __tablename__ = "series_chapter"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    series_tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    anchor_page_id: Mapped[int] = mapped_column(
+        ForeignKey("series_page.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+    )
+    title: Mapped[str | None] = mapped_column(Text, nullable=True)
+    stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

backend/app/models/series_page.py

@@ -1,14 +1,20 @@
 """SeriesPage — ordered image membership for a series-kind Tag.
 
-A series IS a Tag with kind='series'; series_page gives it ordered pages.
-An image belongs to at most one series (UNIQUE image_id). Cover = the
-lowest page_number. page_number is an ordering key only (not unique) —
-reorder rewrites 1..N wholesale.
+A series IS a Tag with kind='series'; series_page gives it a SINGLE flat,
+series-global ordered run of pages (FC-6.x divider reframe). An image belongs to
+at most one series (UNIQUE image_id). Reading order is `page_number` alone — a
+series-wide ordering key (not unique), rewritten 1..N wholesale on reorder so a
+reorder can't transiently collide on an index.
+
+Chapters are cosmetic DIVIDERS anchored to a page (see SeriesChapter); they do
+NOT own pages, so there is no chapter_id here — a page's chapter is derived at
+read time as the nearest preceding divider. stated_page carries the printed page
+number parsed from the source post, nullable when unknown.
 """
 
 from datetime import datetime
 
-from sqlalchemy import DateTime, ForeignKey, Integer, func
+from sqlalchemy import DateTime, ForeignKey, Integer, String, func
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
@@ -26,7 +32,13 @@ class SeriesPage(Base):
         nullable=False,
         unique=True,
     )
-    page_number: Mapped[int] = mapped_column(Integer, nullable=False)
+    # 'placed' = in the series-global run (page_number set); 'pending' = staged
+    # from a post awaiting the operator's sort (page_number NULL). (#789 P2)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, server_default="placed"
+    )
+    page_number: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    stated_page: Mapped[int | None] = mapped_column(Integer, nullable=True)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/series_suggestion.py

@@ -0,0 +1,55 @@
+"""SeriesSuggestion — a confirm-only "this post may continue this series" hint.
+
+The matcher (FC-6.3) scores a (post, candidate series) pair from several weighted
+signals and, above the configured threshold, records a pending suggestion. The
+operator confirms (→ the post is added as a chapter) or dismisses it; FC never
+files a post into a series on its own. status is a plain string (no Postgres
+ENUM — see the check-existing-enums lesson): pending | added | dismissed.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import (
+    JSON,
+    DateTime,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class SeriesSuggestion(Base):
+    __tablename__ = "series_suggestion"
+    __table_args__ = (
+        UniqueConstraint(
+            "post_id", "series_tag_id", name="uq_series_suggestion_post_series"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    post_id: Mapped[int] = mapped_column(
+        ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    series_tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    score: Mapped[float] = mapped_column(Float, nullable=False)
+    signals: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, server_default="pending", index=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

backend/app/models/subscribestar_failed_media.py

@@ -0,0 +1,44 @@
+"""SubscribeStarFailedMedia — per-source dead-letter ledger of SubscribeStar
+media that keeps failing to download/validate.
+
+Mirror of PatreonFailedMedia. Media that fails every walk (404'd CDN URL,
+deleted post, persistently-corrupt bytes) would otherwise re-error forever and
+re-burn backfill chunks. After ``attempts`` reaches the dead-letter threshold
+the ingester skips it on routine tick/backfill walks (recovery still
+re-attempts). A later clean download clears the row.
+
+`filehash` is the same per-media key the seen-ledger uses (CDN content hash or a
+synthesized ``<post_id>:<filename>`` key) — hence String(128). UNIQUE
+(source_id, filehash) is the upsert key.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class SubscribeStarFailedMedia(Base):
+    __tablename__ = "subscribestar_failed_media"
+    __table_args__ = (
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_subscribestar_failed_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    first_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/subscribestar_seen_media.py

@@ -0,0 +1,40 @@
+"""SubscribeStarSeenMedia — per-source ledger of SubscribeStar media already
+downloaded+processed.
+
+Mirror of PatreonSeenMedia for the SubscribeStar native ingester (replacing
+gallery-dl). One queryable row per (source, media) so routine walks skip media
+we've already ingested; recovery mode bypasses the ledger to re-walk.
+
+`filehash` is a CDN content hash when the media URL carries one, else a
+synthesized ``<post_id>:<filename>`` key (SubscribeStar URLs aren't always
+content-addressed) — hence String(128) rather than 32.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class SubscribeStarSeenMedia(Base):
+    __tablename__ = "subscribestar_seen_media"
+    __table_args__ = (
+        # Dedup key the downloader upserts against: one ledger row per
+        # (source, media). A second sighting of the same media is a no-op.
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_subscribestar_seen_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    seen_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/tag_alias.py

@@ -1,6 +1,6 @@
 """TagAlias — maps a model's (name, category) prediction to the operator's
-canonical tag. Resolved at suggestion-read time so raw predictions stay
-unmolested in image_record.tagger_predictions.
+canonical tag. Resolved at suggestion-read time so the raw predictions stored
+in image_prediction stay unmolested.
 """
 
 from datetime import datetime

backend/app/models/tag_allowlist.py

@@ -22,7 +22,11 @@ class TagAllowlist(Base):
     tag_id: Mapped[int] = mapped_column(
         ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
     )
-    min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.95)
+    # Default auto-apply threshold for a newly-accepted tag. 0.90 (lowered from
+    # 0.95 on operator evidence 2026-06-07: 0.95 was too strict and skipped
+    # confident-enough applications). Per-tag value is still tunable in the
+    # allowlist table; existing rows keep whatever they were stored with.
+    min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.90)
     added_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/services/archive_extractor.py

@@ -16,9 +16,45 @@ log = logging.getLogger(__name__)
 
 ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
 
+# Magic-byte signatures, so an archive with a mangled / extension-less filename
+# is still recognised. Patreon attachment download URLs sanitize to names like
+# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk
+# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed
+# those as opaque PostAttachments and NEVER extracted them (operator-flagged
+# 2026-06-06). Detection is by extension first (cheap), then header sniff.
+_RAR_MAGIC = b"Rar!\x1a\x07"
+_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c"
+
+
+def detect_archive_format(path: Path) -> str | None:
+    """Return "zip" | "rar" | "7z" for an archive, else None.
+
+    Trusts a known extension first, then falls back to magic-byte sniffing so a
+    mis-named or extension-less archive is still handled. (zip covers .cbz too.)
+    """
+    ext = Path(path).suffix.lower()
+    if ext in (".zip", ".cbz"):
+        return "zip"
+    if ext == ".rar":
+        return "rar"
+    if ext == ".7z":
+        return "7z"
+    try:
+        if zipfile.is_zipfile(path):
+            return "zip"
+        with open(path, "rb") as fh:
+            head = fh.read(8)
+    except OSError:
+        return None
+    if head.startswith(_RAR_MAGIC):
+        return "rar"
+    if head.startswith(_7Z_MAGIC):
+        return "7z"
+    return None
+
 
 def is_archive(path: Path) -> bool:
-    return Path(path).suffix.lower() in ARCHIVE_EXTS
+    return detect_archive_format(path) is not None
 
 
 @contextmanager
@@ -32,16 +68,16 @@ def extract_archive(path: Path):
     members: list[tuple[str, Path]] = []
     try:
         try:
-            ext = Path(path).suffix.lower()
-            if ext in (".zip", ".cbz"):
+            fmt = detect_archive_format(path)
+            if fmt == "zip":
                 with zipfile.ZipFile(path) as zf:
                     zf.extractall(base)
-            elif ext == ".rar":
+            elif fmt == "rar":
                 import rarfile
 
                 with rarfile.RarFile(path) as rf:
                     rf.extractall(base)
-            elif ext == ".7z":
+            elif fmt == "7z":
                 import py7zr
 
                 with py7zr.SevenZipFile(path, "r") as zf:

backend/app/services/artist_service.py

@@ -10,7 +10,6 @@ from dataclasses import dataclass
 
 from sqlalchemy import and_, case, func, or_, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import (
@@ -24,7 +23,9 @@ from ..models import (
 )
 from ..models.tag import image_tag
 from ..utils.slug import slugify
-from .gallery_service import decode_cursor, encode_cursor, thumbnail_url
+from .db_helpers import get_or_create
+from .gallery_service import thumbnail_url
+from .pagination import decode_cursor, encode_cursor
 
 
 @dataclass(frozen=True)
@@ -250,12 +251,10 @@ class ArtistService:
         )
 
     async def find_or_create(self, name: str) -> tuple[Artist, bool]:
-        """Return (artist, created). Slug-keyed; idempotent under races.
-
-        Audit 2026-06-02: switched from session.rollback() to a
-        begin_nested savepoint + IntegrityError recovery so a lost
-        race doesn't unwind the calling request's surrounding work.
-        Mirrors importer._get_or_create.
+        """Return (artist, created). Slug-keyed; idempotent under races via the
+        shared race-safe db_helpers.get_or_create (savepoint + IntegrityError
+        recovery). A new artist also seeds an ArtistVisit so the directory's
+        `+N new` badge starts at 0.
         """
         cleaned = (name or "").strip()
         if not cleaned:
@@ -263,12 +262,8 @@ class ArtistService:
         slug = slugify(cleaned)
 
         select_existing = select(Artist).where(Artist.slug == slug)
-        existing = (await self.session.execute(select_existing)).scalar_one_or_none()
-        if existing is not None:
-            return existing, False
 
-        sp = await self.session.begin_nested()
-        try:
+        async def _create() -> Artist:
             artist = Artist(name=cleaned, slug=slug)
             self.session.add(artist)
             await self.session.flush()
@@ -279,13 +274,14 @@ class ArtistService:
             # count every image imported in the same session.
             self.session.add(ArtistVisit(artist_id=artist.id))
             await self.session.flush()
-            await sp.commit()
-        except IntegrityError:
-            await sp.rollback()
-            existing = (await self.session.execute(select_existing)).scalar_one()
-            return existing, False
-        await self.session.commit()
-        return artist, True
+            return artist
+
+        artist, created = await get_or_create(
+            self.session, select_existing, _create
+        )
+        if created:
+            await self.session.commit()
+        return artist, created
 
     async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]:
         cleaned = (prefix or "").strip()

backend/app/services/backup_service.py

@@ -15,17 +15,55 @@ lifecycle + soft/hard time limits + retention bookkeeping.
 from __future__ import annotations
 
 import json
+import os
+import shutil
 import subprocess
+import tempfile
 from datetime import UTC, datetime
 from pathlib import Path
 
 _BACKUPS_DIRNAME = "_backups"
 
-# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The
-# Celery soft limit signals the Python process; subprocess.Popen in a
-# blocking syscall ignores that signal. These bound the worst case.
-_DB_SUBPROCESS_TIMEOUT_S = 12 * 60        # 12 min (Celery soft is 10 min)
-_IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60  # 7 hr (Celery soft is 6 hr)
+# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The Celery
+# soft limit signals the Python process; subprocess.Popen in a blocking syscall
+# ignores that signal, so these bound the worst case directly. Each sits just
+# UNDER its task's Celery soft_time_limit so the bounded-kill (_run_bounded) is
+# the primary guard and fires cleanly before Celery's soft/hard limits — which
+# matters because an NFS D-state hang defeats even Celery's SIGKILL (the failure
+# that wedged the maintenance lane for hours, #739).
+#   backup_db_task:     soft=1800s / hard=2100s  → 1700s
+#   backup_images_task: soft=21600s / hard=23400s → 21000s
+_DB_SUBPROCESS_TIMEOUT_S = 1700            # ~28 min, under the 30-min DB soft limit
+_IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft limit
+# Grace after SIGKILL to reap the child. If it can't be reaped in this window
+# (an uninterruptible NFS D-state — the failure mode that wedged the
+# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we
+# STOP waiting and fail fast, freeing the worker slot. The orphan is reaped by
+# the OS once its blocking syscall clears.
+_KILL_REAP_GRACE_S = 10
+
+
+def _run_bounded(cmd: list[str], timeout: int) -> None:
+    """subprocess.run(check=True, timeout) whose reaper can't itself hang.
+
+    subprocess.run's timeout path SIGKILLs the child then blocks in wait() to
+    reap it — but a process stuck in uninterruptible I/O (NFS) can't be reaped,
+    so wait() blocks for hours. Here we bound the post-kill reap and re-raise
+    TimeoutExpired regardless, so the caller fails fast instead of wedging."""
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    try:
+        out, err = proc.communicate(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        try:
+            proc.communicate(timeout=_KILL_REAP_GRACE_S)
+        except subprocess.TimeoutExpired:
+            pass  # unkillable (D-state) — abandon the reap, fail fast
+        raise
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(
+            proc.returncode, cmd, output=out, stderr=err
+        )
 
 
 def _libpq_url(sa_url: str) -> str:
@@ -83,15 +121,29 @@ def backup_db(
     to persist into BackupRun. Raises on subprocess failure."""
     ts = _now_ts()
     out_dir = _backups_dir(images_root)
-    sql_path = out_dir / f"fc_db_{ts}.sql"
-    subprocess.run(
-        [
-            "pg_dump", "--no-owner", "--no-acl",
-            "-f", str(sql_path), _libpq_url(db_url),
-        ],
-        capture_output=True, check=True,
-        timeout=_DB_SUBPROCESS_TIMEOUT_S,
-    )
+    # Custom format (-Fc): compressed (much smaller on NFS) and restored with
+    # pg_restore. The .dump extension marks it as non-SQL. The BackupRun field
+    # is still named sql_path — it's just "the db artifact path".
+    sql_path = out_dir / f"fc_db_{ts}.dump"
+    # Dump to LOCAL disk first, then move the finished file to the (NFS) backups
+    # dir. pg_dump's long phase is then a DB-socket wait + local writes — both
+    # killable — instead of an NFS write that can hang uninterruptibly. Only the
+    # final move touches NFS, and it's a bounded single-file step.
+    fd, tmp_name = tempfile.mkstemp(prefix="fc_db_", suffix=".dump")
+    os.close(fd)
+    tmp_path = Path(tmp_name)
+    try:
+        _run_bounded(
+            [
+                "pg_dump", "--no-owner", "--no-acl", "-Fc",
+                "-f", str(tmp_path), _libpq_url(db_url),
+            ],
+            _DB_SUBPROCESS_TIMEOUT_S,
+        )
+        shutil.move(str(tmp_path), str(sql_path))
+    finally:
+        if tmp_path.exists():
+            tmp_path.unlink(missing_ok=True)
     manifest_path = _write_manifest(
         out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
         artifact_path=sql_path,
@@ -114,15 +166,17 @@ def backup_images(
     ts = _now_ts()
     out_dir = _backups_dir(images_root)
     tar_path = out_dir / f"fc_images_{ts}.tar.zst"
-    subprocess.run(
+    # No local-temp here (the archive is hundreds of GB — it can't stage in
+    # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast
+    # rather than holding the lane for hours.
+    _run_bounded(
         [
             "tar", "--zstd", "-cf", str(tar_path),
             "-C", str(images_root.parent), images_root.name,
             f"--exclude={images_root.name}/_backups",
             f"--exclude={images_root.name}/_quarantine",
         ],
-        capture_output=True, check=True,
-        timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
+        _IMAGES_SUBPROCESS_TIMEOUT_S,
     )
     manifest_path = _write_manifest(
         out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
@@ -139,8 +193,8 @@ def backup_images(
 
 
 def restore_db(*, db_url: str, sql_path: Path) -> None:
-    """Wipe public schema, then load from .sql. Raises on subprocess
-    failure; partial-restore state is the caller's concern."""
+    """Wipe public schema, then load from the custom-format dump. Raises on
+    subprocess failure; partial-restore state is the caller's concern."""
     libpq = _libpq_url(db_url)
     subprocess.run(
         [
@@ -149,8 +203,9 @@ def restore_db(*, db_url: str, sql_path: Path) -> None:
         ],
         capture_output=True, check=True, timeout=120,
     )
+    # Custom-format (-Fc) dumps are restored with pg_restore, not psql.
     subprocess.run(
-        ["psql", libpq, "-f", str(sql_path)],
+        ["pg_restore", "--no-owner", "--no-acl", "-d", libpq, str(sql_path)],
         capture_output=True, check=True,
         timeout=_DB_SUBPROCESS_TIMEOUT_S,
     )

backend/app/services/bulk_tag_service.py

@@ -1,5 +1,7 @@
 """Bulk tag operations over a set of images (Core, set-based, atomic)."""
 
+import math
+
 from sqlalchemy import and_, func, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -13,6 +15,85 @@ class BulkTagService:
     def __init__(self, session: AsyncSession):
         self.session = session
 
+    async def tag_gaps(
+        self, image_ids: list[int], threshold: float
+    ) -> list[dict]:
+        """Cluster-consensus tag gaps (#94 Explore): tags applied to at least
+        `threshold` fraction of the image set but NOT all of it — the
+        "7 of these 10 share Hatsune Miku; these 3 don't" signal.
+
+        For each gap, `missing_image_ids` are the set members that LACK the tag
+        and have NOT rejected it (TagSuggestionRejection) — so apply-to-cluster
+        never re-proposes a tag a neighbor explicitly dismissed. A tag on every
+        image is "common", not a gap; a tag on fewer than 2 isn't consensus.
+        """
+        ids = list({int(i) for i in image_ids})
+        if len(ids) < 2:
+            return []
+        n = len(ids)
+        min_present = max(2, math.ceil(threshold * n))
+        if min_present >= n:
+            return []  # threshold so high only a 100%-common tag could qualify
+
+        cnt = func.count(func.distinct(image_tag.c.image_record_id))
+        gap_rows = (
+            await self.session.execute(
+                select(Tag.id, Tag.name, Tag.kind)
+                .join(image_tag, image_tag.c.tag_id == Tag.id)
+                .where(image_tag.c.image_record_id.in_(ids))
+                .group_by(Tag.id, Tag.name, Tag.kind)
+                .having(and_(cnt >= min_present, cnt < n))
+                .order_by(cnt.desc(), Tag.name.asc())
+            )
+        ).all()
+        if not gap_rows:
+            return []
+        gap_ids = [r.id for r in gap_rows]
+
+        present: dict[int, set[int]] = {}
+        for tid, iid in (
+            await self.session.execute(
+                select(image_tag.c.tag_id, image_tag.c.image_record_id).where(
+                    image_tag.c.tag_id.in_(gap_ids),
+                    image_tag.c.image_record_id.in_(ids),
+                )
+            )
+        ).all():
+            present.setdefault(tid, set()).add(iid)
+
+        rejected: dict[int, set[int]] = {}
+        for tid, iid in (
+            await self.session.execute(
+                select(
+                    TagSuggestionRejection.tag_id,
+                    TagSuggestionRejection.image_record_id,
+                ).where(
+                    TagSuggestionRejection.tag_id.in_(gap_ids),
+                    TagSuggestionRejection.image_record_id.in_(ids),
+                )
+            )
+        ).all():
+            rejected.setdefault(tid, set()).add(iid)
+
+        id_set = set(ids)
+        gaps = []
+        for r in gap_rows:
+            have = present.get(r.id, set())
+            missing = id_set - have - rejected.get(r.id, set())
+            if not missing:
+                continue  # every laggard rejected it — nothing to apply
+            gaps.append(
+                {
+                    "tag_id": r.id,
+                    "name": r.name,
+                    "kind": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
+                    "present_count": len(have),
+                    "total": n,
+                    "missing_image_ids": sorted(missing),
+                }
+            )
+        return gaps
+
     async def common_tags(self, image_ids: list[int]) -> list[dict]:
         """Tags present on EVERY image in image_ids."""
         if not image_ids:

backend/app/services/db_helpers.py

@@ -0,0 +1,52 @@
+"""Shared DB-access helpers for the async services.
+
+`get_or_create` centralizes the race-safe find-or-create dance — SELECT, then on
+a miss a savepoint INSERT that recovers (NOT a full rollback) when a concurrent
+worker inserted the same row first. It was hand-rolled identically in
+ArtistService, TagService and ExtensionService; divergent copies are exactly how
+the duplicate-row / race bugs in [[reference_scalar_one_or_none_duplicates]] crept
+in, so it lives in one place now (DRY pattern sweep 2026-06-09).
+
+Note: this is the ASYNC sibling of `Importer._get_or_create` (sync, used by the
+filesystem-import path). The two can't share an implementation across the
+sync/async boundary; the importer one stays as the lone sync consumer.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from sqlalchemy import Select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+async def get_or_create[T](
+    session: AsyncSession,
+    select_stmt: Select,
+    factory: Callable[[], Awaitable[T]],
+) -> tuple[T, bool]:
+    """Race-safe find-or-create. Returns ``(row, created)``.
+
+    Run ``select_stmt`` (scalar_one_or_none); if a row exists, return it with
+    ``created=False``. Otherwise open a SAVEPOINT and ``await factory()`` — which
+    must add its row(s), flush, and return the primary row. On ``IntegrityError``
+    (a concurrent worker inserted the same row first) roll back the SAVEPOINT —
+    NOT the outer transaction, which would lose the caller's surrounding work —
+    and re-run ``select_stmt`` (scalar_one) to return the row the other worker
+    created. The caller owns the outer commit.
+
+    A UNIQUE/partial-unique constraint matching ``select_stmt``'s predicate is
+    required for the recovery to trip; without it a duplicate slips through.
+    """
+    existing = (await session.execute(select_stmt)).scalar_one_or_none()
+    if existing is not None:
+        return existing, False
+    sp = await session.begin_nested()
+    try:
+        row = await factory()
+        await sp.commit()
+        return row, True
+    except IntegrityError:
+        await sp.rollback()
+        return (await session.execute(select_stmt)).scalar_one(), False

backend/app/services/download_backends.py

@@ -0,0 +1,271 @@
+"""Platform → download-backend dispatch (one place that knows which platforms
+are served by the native FC ingester vs. the gallery-dl subprocess).
+
+gallery-dl wasn't built to be driven by an automated scheduler — no native
+checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
+The native ingester (services/patreon_ingester.py, plan #697) replaces it for
+Patreon and is the path we grow as more platforms migrate. To keep that
+migration DRY, every caller that has to behave differently per backend —
+download routing, the credential-verify probe, cursor handling — asks THIS
+module instead of testing ``platform == "patreon"`` inline. When a platform gets
+a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
+download path and verify switch over together.
+
+The backend surfaces share a UNIFORM signature so a caller invokes the same
+function regardless of platform:
+  - verify_credential(...)  → (ok: bool|None, message: str)
+  - (download stays in download_service for now; uses_native_ingester() is the
+    shared predicate it routes on, so the decision lives here too.)
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from .gallery_dl import DownloadResult, ErrorType
+from .native_ingest_common import NativeIngestError
+from .patreon_ingester import PatreonIngester
+from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
+from .subscribestar_ingester import SubscribeStarIngester
+
+# Platforms whose download + verify go through the native ingester rather than
+# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
+# deviantart) until they migrate too.
+NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
+
+# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
+# messages so the operator sees the exact lookup endpoint that was hit.
+_CAMPAIGNS_API = "https://www.patreon.com/api/campaigns"
+
+
+def _native_ingester_cls(platform: str):
+    """The native ingester class for `platform` (uniform constructor signature).
+    A call-time lookup (not a module-level dict captured at import) so tests can
+    monkeypatch db_mod.PatreonIngester / SubscribeStarIngester and have the
+    dispatch pick up the replacement."""
+    return {
+        "patreon": PatreonIngester,
+        "subscribestar": SubscribeStarIngester,
+    }[platform]
+
+
+def uses_native_ingester(platform: str) -> bool:
+    """True when `platform` is served by the native ingester (not gallery-dl).
+    The single predicate the download path and verify both route on."""
+    return platform in NATIVE_INGESTER_PLATFORMS
+
+
+async def run_download(
+    *,
+    ctx: dict,
+    source_config,
+    skip_value: bool | str,
+    mode: str | None,
+    gdl,
+    sync_session_factory,
+) -> tuple[DownloadResult, str | None]:
+    """Uniform download across backends — the download counterpart to
+    `verify_source_credential`, so this module is the ONE place that knows how
+    each platform both downloads AND verifies (the seam that makes adding a
+    platform a bounded job).
+
+    Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
+    non-None only when a native vanity lookup ran this call (so phase 3 caches
+    it). Native platforms route through their ingester in `mode`
+    (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
+    (download_service) prepares `source_config`/`skip_value`/`mode` from the
+    backfill state machine and owns phase 3.
+    """
+    platform = ctx["platform"]
+    if uses_native_ingester(platform):
+        return await _run_native_ingester(
+            ctx, source_config, mode, gdl, sync_session_factory
+        )
+    result = await gdl.download(
+        url=ctx["url"],
+        artist_slug=ctx["artist_slug"],
+        platform=platform,
+        source_config=source_config,
+        cookies_path=ctx["cookies_path"],
+        auth_token=ctx["auth_token"],
+        skip_value=skip_value,
+    )
+    return result, None
+
+
+async def _resolve_native_campaign_id(
+    platform: str, url: str, cookies_path: str | None, overrides: dict,
+) -> tuple[str | None, str | None]:
+    """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
+    feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
+    campaign id from the vanity URL (resolved non-None when a lookup actually ran,
+    so phase 3 caches it)."""
+    if platform == "subscribestar":
+        return url, None
+    return await resolve_campaign_id_for_source(url, cookies_path, overrides)
+
+
+async def _run_native_ingester(
+    ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
+) -> tuple[DownloadResult, str | None]:
+    """Run the native ingester for a native platform in a worker thread (sync
+    requests/subprocess). Patreon resolves a campaign id from the vanity URL;
+    SubscribeStar's feed id is the creator URL itself. A campaign id we cannot
+    resolve is a loud NOT_FOUND — never a silent empty success.
+
+    `resolved_campaign_id` is non-None only when a lookup ran this call, so phase
+    3 caches it the way the old gallery-dl retry did.
+    """
+    platform = ctx["platform"]
+    overrides = ctx["config_overrides"] or {}
+    campaign_id, resolved_campaign_id = await _resolve_native_campaign_id(
+        platform, ctx["url"], ctx["cookies_path"], overrides
+    )
+    if not campaign_id:
+        # Only reachable for Patreon (SubscribeStar's campaign id is the URL).
+        url = ctx["url"]
+        vanity = extract_vanity(url)
+        return (
+            DownloadResult(
+                success=False,
+                url=url,
+                artist_slug=ctx["artist_slug"],
+                platform=platform,
+                error_type=ErrorType.NOT_FOUND,
+                error_message=(
+                    f"Could not resolve Patreon campaign id. source_url={url!r}; "
+                    f"vanity={vanity!r}; "
+                    f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                    "(vanity lookup failed — cookies expired or creator moved?)"
+                ),
+            ),
+            None,
+        )
+
+    # Honor the operator's existing rate-limit knobs on the native path (plan
+    # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
+    # here on the gdl service) paces media downloads; page fetches use the
+    # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
+    # API-pacing default gallery-dl applied as its `sleep-request`.
+    rate_limit = gdl._rate_limit
+    request_sleep = (
+        source_config.sleep_request
+        if source_config.sleep_request is not None
+        else max(0.5, rate_limit / 4)
+    )
+    ingester = _native_ingester_cls(platform)(
+        images_root=gdl.images_root,
+        cookies_path=ctx["cookies_path"],
+        session_factory=sync_session_factory,
+        validate=gdl._validate_files,
+        rate_limit=rate_limit,
+        request_sleep=request_sleep,
+    )
+    loop = asyncio.get_running_loop()
+    dl_result = await loop.run_in_executor(
+        None,
+        lambda: ingester.run(
+            source_id=ctx["source_id"],
+            campaign_id=campaign_id,
+            artist_slug=ctx["artist_slug"],
+            url=ctx["url"],
+            mode=mode,
+            resume_cursor=source_config.resume_cursor,
+            time_budget_seconds=source_config.timeout,
+            posts_base=int(overrides.get("_backfill_posts", 0)),
+            # plan #709: live progress writes to this running event mid-walk.
+            event_id=ctx.get("event_id"),
+        ),
+    )
+    return dl_result, resolved_campaign_id
+
+
+async def preview_source(
+    *,
+    platform: str,
+    url: str,
+    source_id: int,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    images_root: Path,
+    sync_session_factory,
+    page_limit: int = 3,
+) -> dict:
+    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
+    id, then walk a few pages counting media not already seen/dead — no download.
+
+    Returns the preview dict (total_new / posts_scanned / pages_scanned /
+    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
+    Native-only — the caller gates on `uses_native_ingester`.
+    """
+    import asyncio
+
+    campaign_id, _ = await _resolve_native_campaign_id(
+        platform, url, cookies_path, config_overrides or {}
+    )
+    if not campaign_id:
+        vanity = extract_vanity(url)
+        return {
+            "error": (
+                f"Couldn't resolve the campaign id. source_url={url!r}; "
+                f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                "(cookies expired, or the creator moved/renamed?)."
+            )
+        }
+    ingester = _native_ingester_cls(platform)(
+        images_root=images_root,
+        cookies_path=cookies_path,
+        session_factory=sync_session_factory,
+    )
+    loop = asyncio.get_running_loop()
+    try:
+        result = await loop.run_in_executor(
+            None,
+            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
+        )
+    except NativeIngestError as exc:
+        return {"error": f"Couldn't preview: {exc}"}
+    return result
+
+
+async def verify_source_credential(
+    *,
+    platform: str,
+    url: str,
+    artist_slug: str,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    auth_token: str | None,
+    images_root: Path,
+) -> tuple[bool | None, str]:
+    """Uniform credential probe across backends. Returns `(ok, message)`:
+    True = authenticated, False = rejected, None = inconclusive (drift /
+    network / nothing to test). Callers don't branch on platform — they call
+    this and render the result.
+    """
+    if uses_native_ingester(platform):
+        # Native ingester platforms verify via their own lightweight auth probe
+        # (one authenticated feed fetch). SubscribeStar's probe takes the creator
+        # URL directly; Patreon's resolves the campaign id first.
+        if platform == "subscribestar":
+            from .subscribestar_ingester import verify_subscribestar_credential
+
+            return await verify_subscribestar_credential(url, cookies_path, config_overrides)
+        from .patreon_ingester import verify_patreon_credential
+
+        return await verify_patreon_credential(url, cookies_path, config_overrides)
+
+    # gallery-dl platforms: --simulate one item; the extractor errors before it
+    # can list if auth is bad.
+    from .gallery_dl import GalleryDLService, SourceConfig
+
+    gdl = GalleryDLService(images_root=images_root)
+    return await gdl.verify(
+        url=url,
+        artist_slug=artist_slug,
+        platform=platform,
+        source_config=SourceConfig.from_dict(config_overrides or {}),
+        cookies_path=cookies_path,
+        auth_token=auth_token,
+    )

backend/app/services/download_service.py

@@ -13,7 +13,6 @@ from __future__ import annotations
 
 import asyncio
 import logging
-import re
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -25,44 +24,25 @@ from sqlalchemy.orm import joinedload
 
 from ..models import Artist, DownloadEvent, Source
 from .credential_service import CredentialService
+from .download_backends import run_download, uses_native_ingester
 from .gallery_dl import (
+    BACKFILL_CHUNK_SECONDS,
     BACKFILL_SKIP_VALUE,
-    BACKFILL_TIMEOUT_SECONDS,
     TICK_SKIP_VALUE,
+    DownloadResult,
     ErrorType,
     GalleryDLService,
     SourceConfig,
+    extract_errors_warnings,
+    truncate_log,
 )
 from .importer import Importer
-from .patreon_resolver import resolve_campaign_id
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 
 log = logging.getLogger(__name__)
 
 
-_PATREON_VANITY_RE = re.compile(
-    r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
-    re.IGNORECASE,
-)
-_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
-
-
-def _extract_patreon_vanity(url: str) -> str | None:
-    m = _PATREON_VANITY_RE.match(url)
-    return m.group(1) if m else None
-
-
-def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool:
-    return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
-
-
-def _effective_url(platform: str, source_url: str, overrides: dict) -> str:
-    if platform == "patreon" and overrides.get("patreon_campaign_id"):
-        return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}"
-    return source_url
-
-
 class DownloadService:
     """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
 
@@ -77,12 +57,18 @@ class DownloadService:
         gdl: GalleryDLService,
         importer: Importer,
         cred_service: CredentialService,
+        sync_session_factory=None,
     ):
         self.async_session = async_session
         self.sync_session = sync_session
         self.gdl = gdl
         self.importer = importer
         self.cred_service = cred_service
+        # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED
+        # sessions from for its seen-ledger reads/writes (never one held across
+        # the multi-minute walk — see PatreonIngester). Only the patreon branch
+        # of phase 2 uses it; gallery-dl sources leave it None.
+        self.sync_session_factory = sync_session_factory
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
@@ -107,63 +93,91 @@ class DownloadService:
         self.sync_session.close()
 
         source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
-        # alembic 0031 / plan #544: derive skip_value + timeout from the
-        # source's backfill_runs_remaining counter. When > 0, walk the full
-        # post history (skip: True + 1170s); when 0, exit gallery-dl after
-        # 20 contiguous archived items (skip: "exit:20" + the default
-        # 870s). Operator sets backfill via POST /api/sources/{id}/backfill.
-        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
-        if backfill_remaining > 0:
+        # Backfill mode (plan #693): the source's `_backfill_state == "running"`
+        # selects a time-boxed deep-walk chunk — skip: True (walk full history)
+        # + the BACKFILL_CHUNK_SECONDS budget, resuming from the cursor
+        # checkpoint (plan #689). State is the single source of truth; it stays
+        # "running" across ticks until the walk reaches the bottom (phase 3
+        # flips it to "complete"). Otherwise tick mode: exit gallery-dl after
+        # 20 contiguous archived items (skip: "exit:20" + the default 870s).
+        # Operator drives this via POST /api/sources/{id}/backfill {action}.
+        overrides = ctx["config_overrides"] or {}
+        in_backfill = overrides.get("_backfill_state") == "running"
+        # Recovery (plan #697) reuses the entire #693 backfill state machine —
+        # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and
+        # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted
+        # near-dups get re-fetched and re-evaluated under the CURRENT pHash
+        # threshold (tier-2 disk still spares files we kept). The
+        # `_backfill_bypass_seen` flag rides alongside the running backfill state;
+        # download mode is "recovery" when both are set.
+        bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
+        # #830 recapture: re-grab post bodies/links + localize on-disk inline
+        # images, WITHOUT re-downloading media. Rides alongside the backfill
+        # state like _backfill_bypass_seen; mutually exclusive with it.
+        recapture = bool(overrides.get("_backfill_recapture"))
+        if in_backfill:
             skip_value: bool | str = BACKFILL_SKIP_VALUE
-            source_config.timeout = BACKFILL_TIMEOUT_SECONDS
+            source_config.timeout = BACKFILL_CHUNK_SECONDS
+            pending_cursor = overrides.get("_backfill_cursor")
+            if uses_native_ingester(ctx["platform"]) and pending_cursor:
+                source_config.resume_cursor = pending_cursor
         else:
             skip_value = TICK_SKIP_VALUE
 
-        effective_url = _effective_url(
-            ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
+        # Phase 2 dispatch is uniform across backends (download_backends.
+        # run_download — the download counterpart to verify_source_credential):
+        # native platforms run their ingester in `mode` (zero per-file HEADs,
+        # native cursor/resume, loud drift detection); gallery-dl platforms run
+        # the subprocess. Either returns a DownloadResult-shaped object so phase 3
+        # is untouched. `mode` is None for gallery-dl (run_download ignores it).
+        mode: str | None = None
+        if uses_native_ingester(ctx["platform"]):
+            if in_backfill and bypass_seen:
+                mode = "recovery"
+            elif in_backfill and recapture:
+                mode = "recapture"
+            elif in_backfill:
+                mode = "backfill"
+            else:
+                mode = "tick"
+        dl_result, resolved_campaign_id = await self._run_download(
+            ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode,
         )
 
-        dl_result = await self.gdl.download(
-            url=effective_url,
-            artist_slug=ctx["artist_slug"],
-            platform=ctx["platform"],
-            source_config=source_config,
-            cookies_path=ctx["cookies_path"],
-            auth_token=ctx["auth_token"],
-            skip_value=skip_value,
-        )
-
-        resolved_campaign_id: str | None = None
-        if (
-            ctx["platform"] == "patreon"
-            and not dl_result.success
-            and "patreon_campaign_id" not in (ctx["config_overrides"] or {})
-            and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr)
-        ):
-            vanity = _extract_patreon_vanity(ctx["url"])
-            if vanity:
-                log.info(
-                    "Attempting campaign-ID resolution for %s (%s)",
-                    ctx["artist_slug"], vanity,
+        # A backfill chunk that hit its time-box but made forward progress is
+        # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as
+        # "ok/progress" (PARTIAL maps to status "ok"), not a red error, and the
+        # next chunk just resumes from the new cursor (plan #693). A chunk that
+        # timed out with NO progress stays TIMEOUT and feeds phase 3's
+        # stall-guard. Leave RATE_LIMITED alone so the platform-cooldown fires.
+        if in_backfill and dl_result.error_type == ErrorType.TIMEOUT:
+            new_cursor = dl_result.cursor  # plan #704: structured, not scraped
+            advanced = bool(
+                (new_cursor and new_cursor != overrides.get("_backfill_cursor"))
+                or dl_result.files_downloaded > 0
+            )
+            if advanced:
+                dl_result.error_type = ErrorType.PARTIAL
+                dl_result.error_message = (
+                    f"Backfill chunk: {dl_result.files_downloaded} file(s) — continuing"
                 )
-                resolved_campaign_id = await resolve_campaign_id(
-                    vanity, ctx["cookies_path"]
-                )
-                if resolved_campaign_id:
-                    dl_result = await self.gdl.download(
-                        url=f"https://www.patreon.com/id:{resolved_campaign_id}",
-                        artist_slug=ctx["artist_slug"],
-                        platform=ctx["platform"],
-                        source_config=source_config,
-                        cookies_path=ctx["cookies_path"],
-                        auth_token=ctx["auth_token"],
-                        skip_value=skip_value,
-                    )
 
         return await self._phase3_persist(
             ctx["event_id"], ctx, dl_result, resolved_campaign_id,
         )
 
+    async def _run_download(
+        self, *, ctx: dict, source_config, skip_value, mode: str | None,
+    ) -> tuple[DownloadResult, str | None]:
+        """Phase-2 dispatch → `download_backends.run_download`, passing this
+        service's gdl + sync sessionmaker. Kept as a thin instance method so tests
+        can stub the whole phase-2 dispatch on the service, and so the per-backend
+        construction lives in download_backends (the backend registry)."""
+        return await run_download(
+            ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode,
+            gdl=self.gdl, sync_session_factory=self.sync_session_factory,
+        )
+
     async def _phase1_setup(self, source_id: int) -> dict[str, Any]:
         source = (await self.async_session.execute(
             select(Source).options(joinedload(Source.artist)).where(Source.id == source_id)
@@ -260,7 +274,20 @@ class DownloadService:
             artist = self.sync_session.get(Artist, ctx["artist_id"])
         source_row = self.sync_session.get(Source, ctx["source_id"])
 
+        # Post-first (#856): on the native ingester the post-record is the sole
+        # writer of the post body/links, so the per-media import must NOT apply post
+        # fields. gallery-dl platforms keep writing them via the sidecar. The
+        # Importer is per-task, so this per-instance flag is safe. uses_native_ingester
+        # is the future-proof seam — a platform migrating onto the native core
+        # flips to post-first automatically (milestone #67, step 3).
+        self.importer.post_first = uses_native_ingester(ctx["platform"])
+
         import_summary = {"attached": 0, "skipped": 0, "errors": 0}
+        # Archives detected but captured WITHOUT extracting any image (probe
+        # rejected / corrupt / missing extractor backend). Surfaced on the event
+        # so a post showing "no images" beside a zip is diagnosable (plan
+        # follow-up 2026-06-06 — the recurring archive-association report).
+        unextracted_archives: list[dict] = []
         bytes_downloaded = 0
 
         loop = asyncio.get_running_loop()
@@ -314,6 +341,17 @@ class DownloadService:
                 # mirror duplicate_hash cleanup so we don't keep two copies.
                 # Operator-flagged 2026-06-02 (Lustria OST zip).
                 import_summary["attached"] += 1
+                # An archive captured WITHOUT extracting any image carries a
+                # reason on result.error — record it so the event explains the
+                # "no images beside a zip" symptom instead of staying silent.
+                if result.error:
+                    unextracted_archives.append(
+                        {"file": path.name, "reason": result.error}
+                    )
+                    log.warning(
+                        "archive captured unextracted (%s): %s",
+                        path.name, result.error,
+                    )
                 try:
                     bytes_downloaded += path.stat().st_size  # noqa: ASYNC240
                 except OSError:
@@ -356,15 +394,68 @@ class DownloadService:
             else:
                 import_summary["errors"] += 1
 
+        # Post-only records: media-less posts (pure-text) the native ingester
+        # captured so the artist archive is complete. Upsert each (keyed on
+        # external_post_id → updates the same Post a media import would create,
+        # never doubles). No file to clean up; the sidecar stays on disk.
+        for rec_str in getattr(dl_result, "post_record_paths", None) or []:
+            rec_path = Path(rec_str)
+            if not rec_path.exists():  # noqa: ASYNC240
+                continue
+
+            def _upsert(p=rec_path):
+                return self.importer.upsert_post_record(
+                    p, artist=artist, source=source_row,
+                )
+
+            await loop.run_in_executor(None, _upsert)
+
+        # #830 recapture: backfill source_filehash on EXISTING on-disk images so
+        # their post-body inline <img src=CDN> remaps to the local copy. A
+        # SEPARATE non-deleting channel (NOT the import list — that would unlink
+        # the duplicate file). Empty outside recapture mode.
+        relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
+        relinked = 0
+        for rel_str, rel_url in relink_pairs:
+            rel_path = Path(rel_str)
+            if not rel_path.exists():  # noqa: ASYNC240
+                continue
+
+            def _relink(p=rel_path, u=rel_url):
+                return self.importer.relink_source_filehash(p, u, artist=artist)
+
+            if await loop.run_in_executor(None, _relink):
+                relinked += 1
+        if relink_pairs:
+            # recapture diagnostic: how many on-disk images got their
+            # source_filehash backfilled (inline-image localization). < total is
+            # normal — files already carrying a filehash are skipped (NULL-only).
+            log.info(
+                "recap: relinked source_filehash on %d/%d on-disk image(s)",
+                relinked, len(relink_pairs),
+            )
+
+        # Kick the off-platform file-host downloader for any links this run
+        # recorded (mega/gdrive/…). Global + idempotent (only claims pending/
+        # retryable rows); the beat sweep is the backstop. Lazy import dodges a
+        # task-module import cycle.
+        from ..tasks.external import sweep_external_links
+        sweep_external_links.delay()
+
         ev = (await self.async_session.execute(
             select(DownloadEvent).where(DownloadEvent.id == event_id)
         )).scalar_one()
 
-        run_stats = self.gdl._compute_run_stats(
-            dl_result.return_code, dl_result.stdout, dl_result.stderr
-        )
+        # plan #704: the native ingester returns structured run_stats; only the
+        # gallery-dl path needs the regex-over-stdout reconstruction.
+        if dl_result.run_stats is not None:
+            run_stats = dict(dl_result.run_stats)
+        else:
+            run_stats = self.gdl._compute_run_stats(
+                dl_result.return_code, dl_result.stdout, dl_result.stderr
+            )
         run_stats["quarantined_count"] = dl_result.files_quarantined
-        stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
+        stderr_summary = extract_errors_warnings(dl_result.stderr)
 
         # Plan #544: PARTIAL means the run downloaded ≥1 file but the
         # subprocess didn't finish in budget (typically wall-clock timeout
@@ -384,50 +475,118 @@ class DownloadService:
         ev.metadata_ = {
             "run_stats": run_stats,
             "error_type": dl_result.error_type.value if dl_result.error_type else None,
-            "stdout": self.gdl._truncate_log(dl_result.stdout) or None,
-            "stderr": self.gdl._truncate_log(dl_result.stderr) or None,
+            "stdout": truncate_log(dl_result.stdout) or None,
+            "stderr": truncate_log(dl_result.stderr) or None,
             "stderr_errors_warnings": stderr_summary or None,
             "duration_seconds": dl_result.duration_seconds,
             "quarantined_paths": dl_result.quarantined_paths or None,
             "import_summary": import_summary,
+            # Archives detected but captured without extracting an image — the
+            # recurring "post shows a zip but no images" report. Each entry is
+            # {file, reason}; None when every archive extracted cleanly.
+            "unextracted_archives": unextracted_archives or None,
         }
         await self._update_source_health(
             source_id=ctx["source_id"], status=status, error_message=ev.error,
             error_type=dl_result.error_type.value if dl_result.error_type else None,
+            retry_after_seconds=getattr(dl_result, "retry_after_seconds", None),
         )
-        # Plan #544: backfill lifecycle — auto-complete when a clean
-        # backfill run drained the queue (gallery-dl exited 0 + zero files
-        # downloaded means there was nothing to fetch); otherwise decrement
-        # the counter. Next tick falls back to tick mode once it hits 0.
-        #
-        # Audit 2026-06-02 gating: VALIDATION_FAILED also exits the
-        # subprocess with return_code=0 and files_downloaded=0 (every
-        # file was quarantined), which used to match the auto-complete
-        # predicate exactly — zeroing the operator's armed budget on
-        # the FIRST quarantine run instead of decrementing. Require
-        # dl_result.success + no error_type so only genuinely-empty
-        # successful runs drain the counter.
-        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
-        if backfill_remaining > 0:
-            src = (await self.async_session.execute(
-                select(Source).where(Source.id == ctx["source_id"])
-            )).scalar_one()
-            queue_drained = (
-                dl_result.success
-                and dl_result.error_type is None
-                and dl_result.return_code == 0
-                and dl_result.files_downloaded == 0
-            )
-            if queue_drained:
-                src.backfill_runs_remaining = 0
-            else:
-                src.backfill_runs_remaining = max(0, backfill_remaining - 1)
+        await self._apply_backfill_lifecycle(ctx, dl_result)
         await self.async_session.commit()
         return event_id
 
+    async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
+        """Backfill state machine (plan #693, building on the cursor of #689).
+
+        A backfill runs in time-boxed chunks while
+        `config_overrides["_backfill_state"] == "running"`. Each chunk:
+          - COMPLETES the walk → clean rc=0 (gallery-dl with skip:True exits 0
+            only after exhausting the newest→oldest walk; a chunk cut short by
+            its time-box returns success=False / rc<0 via TimeoutExpired). On
+            completion: state="complete", clear the cursor, return to tick mode.
+          - made PROGRESS (cursor advanced and/or files written) → stay
+            "running", checkpoint the new cursor, bump the chunk counter, and
+            spend one of the safety-cap chunks (backfill_runs_remaining). If the
+            cap is exhausted without finishing → state="stalled".
+          - made NO progress → increment the stall counter; two strikes →
+            state="stalled", clear the cursor (a wedged walk can't loop).
+
+        State is the single source of truth; the cursor lives only inside a
+        running backfill. config_overrides is reassigned (not mutated in place)
+        for SQLAlchemy JSON change detection.
+        """
+        overrides = ctx["config_overrides"] or {}
+        if overrides.get("_backfill_state") != "running":
+            return  # not backfilling — tick mode, nothing to do
+
+        old_cursor = overrides.get("_backfill_cursor")
+        cap_remaining = ctx.get("backfill_runs_remaining", 0) or 0
+
+        src = (await self.async_session.execute(
+            select(Source).where(Source.id == ctx["source_id"])
+        )).scalar_one()
+        new_overrides = dict(src.config_overrides or {})
+        chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1
+        new_overrides["_backfill_chunks"] = chunks
+        # plan #704 (#5): _backfill_posts (the live progress badge) is OWNED by the
+        # ingester now — it writes a monotonic absolute mid-walk at each page
+        # boundary (ingest_core._checkpoint_posts), so the badge climbs DURING a
+        # chunk instead of jumping once per chunk here, and the re-walked resume
+        # page is no longer double-counted. new_overrides (read fresh above)
+        # carries the ingester's committed value forward untouched.
+
+        completed = (
+            dl_result.success
+            and dl_result.error_type is None
+            and dl_result.return_code == 0
+        )
+        if completed:
+            new_overrides["_backfill_state"] = "complete"
+            new_overrides.pop("_backfill_cursor", None)
+            new_overrides.pop("_backfill_cursor_stalls", None)
+            # plan #697: a recovery walk shares this lifecycle; clear its bypass
+            # flag on completion so the next routine tick honors the seen-ledger.
+            new_overrides.pop("_backfill_bypass_seen", None)
+            # #830: same for the recapture flag.
+            new_overrides.pop("_backfill_recapture", None)
+            src.config_overrides = new_overrides
+            src.backfill_runs_remaining = 0
+            return
+
+        # Did not finish. The native ingester checkpoints + resumes via cursor
+        # (carried structurally on the result, plan #704); gallery-dl platforms
+        # have no resumable cursor (every chunk re-walks from the top), so they
+        # advance only by the download archive growing.
+        new_cursor = (
+            dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
+        )
+        advanced = bool(
+            (new_cursor and new_cursor != old_cursor)
+            or dl_result.files_downloaded > 0
+        )
+        if advanced:
+            if new_cursor:
+                new_overrides["_backfill_cursor"] = new_cursor
+            new_overrides.pop("_backfill_cursor_stalls", None)
+            cap_remaining = max(0, cap_remaining - 1)
+            src.backfill_runs_remaining = cap_remaining
+            if cap_remaining == 0:
+                # Safety cap hit before reaching the bottom — pause, don't loop.
+                new_overrides["_backfill_state"] = "stalled"
+        else:
+            stalls = int(new_overrides.get("_backfill_cursor_stalls", 0)) + 1
+            if stalls >= 2:
+                new_overrides["_backfill_state"] = "stalled"
+                new_overrides.pop("_backfill_cursor", None)
+                new_overrides.pop("_backfill_cursor_stalls", None)
+            else:
+                new_overrides["_backfill_cursor_stalls"] = stalls
+
+        src.config_overrides = new_overrides
+
     async def _update_source_health(
         self, *, source_id: int, status: str, error_message: str | None,
-        error_type: str | None = None,
+        error_type: str | None = None, retry_after_seconds: float | None = None,
     ) -> None:
         """FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}.
 
@@ -459,7 +618,17 @@ class DownloadService:
             # by error class without opening Logs per row.
             source.error_type = error_type
             if error_type == "rate_limited":
-                await set_platform_cooldown(self.async_session, source.platform)
+                # plan #708 B1: honor the server's Retry-After when the native
+                # client surfaced one (clamped to a sane [60, 3600] window so a
+                # tiny hint can't leave the platform effectively un-cooled and a
+                # huge one can't strand it for hours); else the flat default.
+                if retry_after_seconds is not None:
+                    seconds = int(min(max(retry_after_seconds, 60), 3600))
+                    await set_platform_cooldown(
+                        self.async_session, source.platform, seconds=seconds,
+                    )
+                else:
+                    await set_platform_cooldown(self.async_session, source.platform)
         elif status == "skipped":
             source.last_error = None
         source.last_checked_at = now

backend/app/services/extension_service.py

@@ -11,12 +11,12 @@ from __future__ import annotations
 import re
 
 from sqlalchemy import select
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import Artist, Source
 from ..utils.slug import slugify
-from .source_service import NEW_SOURCE_BACKFILL_RUNS
+from .db_helpers import get_or_create
+from .source_service import BACKFILL_MAX_CHUNKS
 
 
 class UnknownPlatformError(Exception):
@@ -169,24 +169,16 @@ class ExtensionService:
         500 against uq_artist_slug.
         """
         slug = slugify(raw_name)
-        existing = (await self.session.execute(
-            select(Artist).where(Artist.slug == slug)
-        )).scalar_one_or_none()
-        if existing is not None:
-            return existing, False
-        sp = await self.session.begin_nested()
-        try:
+
+        async def _create() -> Artist:
             artist = Artist(name=raw_name, slug=slug, is_subscription=True)
             self.session.add(artist)
             await self.session.flush()
-            await sp.commit()
-            return artist, True
-        except IntegrityError:
-            await sp.rollback()
-            recovered = (await self.session.execute(
-                select(Artist).where(Artist.slug == slug)
-            )).scalar_one()
-            return recovered, False
+            return artist
+
+        return await get_or_create(
+            self.session, select(Artist).where(Artist.slug == slug), _create
+        )
 
     async def _find_or_create_source(
         self, *, artist_id: int, platform: str, url: str,
@@ -194,40 +186,32 @@ class ExtensionService:
         """Race-safe — same pattern as _find_or_create_artist above. The
         uq_source_artist_platform_url constraint catches the duplicate
         insert; we roll the savepoint back and re-select."""
-        existing = (await self.session.execute(
-            select(Source).where(
-                Source.artist_id == artist_id,
-                Source.platform == platform,
-                Source.url == url,
-            )
-        )).scalar_one_or_none()
-        if existing is not None:
-            return existing, False
-        sp = await self.session.begin_nested()
-        try:
-            # New subscription sources arm a few backfill runs so the
-            # first ticks walk the full history (otherwise gallery-dl's
-            # exit:20 short-circuits before the archive is built).
-            # Mirrors SourceService.create — without it, Firefox quick-
-            # add on a creator with >20 unsynced posts would surface
-            # as "check failed" with no diagnosis. Audit 2026-06-02.
+        select_existing = select(Source).where(
+            Source.artist_id == artist_id,
+            Source.platform == platform,
+            Source.url == url,
+        )
+
+        async def _create() -> Source:
+            # New subscription sources arm run-until-done backfill (plan #693)
+            # so the first ticks walk the full history (otherwise gallery-dl's
+            # exit:20 short-circuits before the archive is built). Mirrors
+            # SourceService.create — without it, Firefox quick-add on a creator
+            # with >20 unsynced posts would surface as "check failed" with no
+            # diagnosis. Audit 2026-06-02.
             src = Source(
                 artist_id=artist_id, platform=platform,
                 url=url, enabled=True,
-                backfill_runs_remaining=NEW_SOURCE_BACKFILL_RUNS,
+                config_overrides={"_backfill_state": "running"},
+                backfill_runs_remaining=BACKFILL_MAX_CHUNKS,
             )
             self.session.add(src)
             await self.session.flush()
-            await sp.commit()
-        except IntegrityError:
-            await sp.rollback()
-            recovered = (await self.session.execute(
-                select(Source).where(
-                    Source.artist_id == artist_id,
-                    Source.platform == platform,
-                    Source.url == url,
-                )
-            )).scalar_one()
-            return recovered, False
-        await self.session.commit()
-        return src, True
+            return src
+
+        src, created = await get_or_create(
+            self.session, select_existing, _create
+        )
+        if created:
+            await self.session.commit()
+        return src, created

backend/app/services/external_fetch.py

@@ -0,0 +1,292 @@
+"""Fetchers for off-platform file hosts (mega / gdrive / mediafire / dropbox /
+pixeldrain).
+
+A shared, reusable subsystem: given an external_link URL, fetch the file(s) into
+a destination directory and report the outcome. The download worker (separate
+slice) drives these off the external_link ledger; any in-house downloader can
+call `fetch_external()` directly.
+
+No single tool covers all five hosts, so a small registry maps host → fetch
+function behind one signature:
+
+    fetch_external(host, url, dest_dir, *, read_timeout, total_timeout, should_stop)
+        -> FetchResult
+
+Backends:
+  - dropbox    : force the direct-download variant (dl=1) + stream GET.
+  - pixeldrain : GET the /api/file/{id} endpoint.
+  - mediafire  : scrape the download page for the direct link + stream GET.
+  - gdrive     : gdown (handles the confirm-token + virus-scan interstitial).
+  - mega       : `megatools dl` subprocess (public link incl. #key); needs the
+                 `megatools` binary in the runtime image (Debian apt package).
+
+Public links work credential-free (rule 26); per-host creds are a later Settings
+concern. Plain-HTTP homelab — no secure-context API. The HTTP / gdown /
+subprocess calls go through module-level seams so unit tests run without
+network, gdown, or MEGAcmd.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+import subprocess
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import requests
+
+log = logging.getLogger(__name__)
+
+_CHUNK = 1 << 16
+# Two distinct limits, because conflating them (the old single 3000s value) meant
+# a stalled HTTP connection tied up a download-worker slot + the per-host lock for
+# ~50 min before failing (operator-flagged 2026-06-17):
+#   * READ timeout — max idle gap between bytes on an HTTP socket. A stalled host
+#     (socket open, nothing flowing) is the common failure mode; a short read
+#     timeout fails it fast. This is what requests' `timeout` actually enforces —
+#     per-read, never a total.
+#   * TOTAL budget — generous wall-clock cap for a file that IS actively
+#     transferring (big films/packs). Enforced as a deadline across chunks, since
+#     no HTTP client timeout bounds the total. Also the subprocess total for mega.
+_CONNECT_TIMEOUT = 30.0
+_READ_TIMEOUT = 60.0
+_TOTAL_TIMEOUT = 1800.0  # 30 min per fetch
+_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+)
+# MediaFire's download page embeds the real file URL in a download-button href.
+_MEDIAFIRE_RE = re.compile(
+    r'href="(https://download[^"]+?\.mediafire\.com/[^"]+)"', re.IGNORECASE
+)
+_CD_FILENAME_RE = re.compile(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', re.IGNORECASE)
+
+
+@dataclass
+class FetchResult:
+    files: list[Path] = field(default_factory=list)
+    bytes: int = 0
+    error: str | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None and bool(self.files)
+
+
+class ExternalFetchError(Exception):
+    """A fetch failed in a way worth recording on the link's last_error."""
+
+
+# -- seams (monkeypatched in tests) ----------------------------------------
+
+def _http_get(url: str, *, timeout: float, headers: dict | None = None,
+              stream: bool = True) -> requests.Response:
+    hdrs = {"User-Agent": _USER_AGENT}
+    if headers:
+        hdrs.update(headers)
+    return requests.get(url, timeout=timeout, headers=hdrs, stream=stream)
+
+
+def _gdown_download(url: str, out_dir: str) -> str | None:
+    """Download a Google Drive url into out_dir via gdown; return the written
+    path. Imported lazily so the dep is optional at module-import time."""
+    # Lazy import: keeps gdown an optional dep that isn't needed to import this
+    # module (e.g. the no-DB unit lane that never exercises a real fetch).
+    import gdown
+    # A trailing sep tells gdown to keep the server-side filename inside out_dir.
+    return gdown.download(url, output=out_dir + os.sep, quiet=True, fuzzy=True)
+
+
+def _run_mega_get(url: str, out_dir: str, *, timeout: float) -> None:
+    """Download a mega.nz public link (key in the #fragment) into out_dir via
+    `megatools dl` (the Debian `megatools` package). Raises ExternalFetchError
+    on non-zero exit."""
+    # Fixed argv (not shell): only `url` is external input, passed positionally,
+    # so there's no shell-injection surface.
+    proc = subprocess.run(
+        ["megatools", "dl", "--path", out_dir, url],
+        capture_output=True, text=True, timeout=timeout, check=False,
+    )
+    if proc.returncode != 0:
+        raise ExternalFetchError(
+            f"megatools dl exit {proc.returncode}: {(proc.stderr or '').strip()[:300]}"
+        )
+
+
+# -- helpers ---------------------------------------------------------------
+
+def _safe_name(name: str, fallback: str) -> str:
+    name = os.path.basename((name or "").strip().strip('"'))
+    # Strip path separators / control chars; never empty.
+    name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", name).strip(". ")
+    return name or fallback
+
+
+def _filename_from(resp: requests.Response, url: str, fallback: str) -> str:
+    cd = resp.headers.get("Content-Disposition", "")
+    m = _CD_FILENAME_RE.search(cd)
+    if m:
+        return _safe_name(m.group(1), fallback)
+    path_name = os.path.basename(urlsplit(url).path)
+    return _safe_name(path_name, fallback)
+
+
+def _stream_to_file(resp: requests.Response, dest: Path,
+                    should_stop: Callable[[], bool],
+                    *, deadline: float | None = None) -> int:
+    """Stream a response body to `dest` (atomic via .part). Returns byte count.
+    Honors should_stop and the total-budget `deadline` (a time.monotonic() value)
+    between chunks; the partial file is removed on either abort."""
+    part = dest.with_name(dest.name + ".part")
+    total = 0
+    try:
+        with part.open("wb") as fh:
+            for chunk in resp.iter_content(chunk_size=_CHUNK):
+                if should_stop():
+                    raise ExternalFetchError("stopped")
+                if deadline is not None and time.monotonic() > deadline:
+                    raise ExternalFetchError("exceeded total fetch budget")
+                if chunk:
+                    fh.write(chunk)
+                    total += len(chunk)
+    except BaseException:
+        part.unlink(missing_ok=True)
+        raise
+    os.replace(part, dest)
+    return total
+
+
+def _get_to_dir(url: str, dest_dir: Path, *, read_timeout: float,
+                total_timeout: float, should_stop: Callable[[], bool],
+                fallback: str, headers: dict | None = None) -> FetchResult:
+    # (connect, read): a short read timeout fails a stalled socket fast; the total
+    # budget is enforced separately as a deadline across chunks (requests has no
+    # total-download timeout).
+    resp = _http_get(
+        url, timeout=(_CONNECT_TIMEOUT, read_timeout), headers=headers, stream=True
+    )
+    if resp.status_code != 200:
+        return FetchResult(error=f"HTTP {resp.status_code} for {url}")
+    name = _filename_from(resp, url, fallback)
+    dest = dest_dir / name
+    written = _stream_to_file(
+        resp, dest, should_stop, deadline=time.monotonic() + total_timeout
+    )
+    return FetchResult(files=[dest], bytes=written)
+
+
+# -- per-host fetchers -----------------------------------------------------
+
+def _fetch_dropbox(url: str, dest_dir: Path, *, read_timeout: float,
+                   total_timeout: float,
+                   should_stop: Callable[[], bool]) -> FetchResult:
+    # Force the direct-download variant: dl=1 (Dropbox serves an HTML preview
+    # for dl=0). Rewrite/insert the param rather than string-replace so ?dl=0,
+    # &dl=0, and a missing param all resolve.
+    parts = urlsplit(url)
+    q = dict(parse_qsl(parts.query))
+    q["dl"] = "1"
+    direct = urlunsplit(parts._replace(query=urlencode(q)))
+    return _get_to_dir(direct, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback="dropbox-file")
+
+
+def _fetch_pixeldrain(url: str, dest_dir: Path, *, read_timeout: float,
+                      total_timeout: float,
+                      should_stop: Callable[[], bool]) -> FetchResult:
+    # /u/{id} (and /l/{id}) → the API file endpoint.
+    file_id = urlsplit(url).path.rstrip("/").split("/")[-1]
+    if not file_id:
+        return FetchResult(error=f"no pixeldrain id in {url}")
+    api = f"https://pixeldrain.com/api/file/{file_id}"
+    return _get_to_dir(api, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback=f"{file_id}.bin")
+
+
+def _fetch_mediafire(url: str, dest_dir: Path, *, read_timeout: float,
+                     total_timeout: float,
+                     should_stop: Callable[[], bool]) -> FetchResult:
+    page = _http_get(url, timeout=(_CONNECT_TIMEOUT, read_timeout), stream=False)
+    if page.status_code != 200:
+        return FetchResult(error=f"HTTP {page.status_code} for mediafire page")
+    m = _MEDIAFIRE_RE.search(page.text or "")
+    if not m:
+        return FetchResult(error="mediafire direct link not found on page")
+    return _get_to_dir(m.group(1), dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop,
+                       fallback="mediafire-file")
+
+
+def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
+                  total_timeout: float,
+                  should_stop: Callable[[], bool]) -> FetchResult:
+    # gdown manages its own HTTP session/timeouts; the task's celery hard limit is
+    # the outer backstop. read_timeout/total_timeout are accepted for a uniform
+    # registry signature but not separately enforceable here.
+    out = _gdown_download(url, str(dest_dir))
+    if not out:
+        return FetchResult(error="gdown returned no file (quota / private?)")
+    p = Path(out)
+    if not p.exists():
+        return FetchResult(error=f"gdown reported {out} but it is missing")
+    return FetchResult(files=[p], bytes=p.stat().st_size)
+
+
+def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
+                total_timeout: float,
+                should_stop: Callable[[], bool]) -> FetchResult:
+    before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
+    # megatools is a subprocess: its timeout IS a total wall-clock cap (the read
+    # timeout has no analogue here), so the total budget applies directly.
+    _run_mega_get(url, str(dest_dir), timeout=total_timeout)
+    new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
+    if not new:
+        return FetchResult(error="mega-get wrote no new file")
+    return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))
+
+
+_REGISTRY: dict[str, Callable[..., FetchResult]] = {
+    "dropbox": _fetch_dropbox,
+    "pixeldrain": _fetch_pixeldrain,
+    "mediafire": _fetch_mediafire,
+    "gdrive": _fetch_gdrive,
+    "mega": _fetch_mega,
+}
+
+SUPPORTED_HOSTS = tuple(_REGISTRY)
+
+
+def fetch_external(host: str, url: str, dest_dir: Path, *,
+                   read_timeout: float = _READ_TIMEOUT,
+                   total_timeout: float = _TOTAL_TIMEOUT,
+                   should_stop: Callable[[], bool] = lambda: False) -> FetchResult:
+    """Fetch `url` (a `host` link) into `dest_dir`. Returns a FetchResult; never
+    raises — any backend error (transport, read/total timeout, non-200, scrape
+    miss, subprocess failure, stop) is captured on `.error` so the worker can
+    record it and move on.
+
+    `read_timeout` fails a stalled HTTP socket fast (idle gap between bytes);
+    `total_timeout` is the generous wall-clock cap for a large file that is
+    actively transferring (and the subprocess total for mega)."""
+    fetcher = _REGISTRY.get(host)
+    if fetcher is None:
+        return FetchResult(error=f"unsupported host {host!r}")
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        return fetcher(url, dest_dir, read_timeout=read_timeout,
+                       total_timeout=total_timeout, should_stop=should_stop)
+    except requests.RequestException as exc:
+        return FetchResult(error=f"transport error: {exc}")
+    except subprocess.TimeoutExpired:
+        return FetchResult(error="timed out")
+    except ExternalFetchError as exc:
+        return FetchResult(error=str(exc))
+    except Exception as exc:  # never let a backend quirk kill the worker
+        log.warning("external fetch (%s) failed for %s: %s", host, url, exc)
+        return FetchResult(error=f"{type(exc).__name__}: {exc}")

backend/app/services/file_validator.py

@@ -11,9 +11,15 @@ expected to write.
 
 from __future__ import annotations
 
+import json
+import logging
+import shutil
 from dataclasses import dataclass
+from datetime import UTC, datetime
 from pathlib import Path
 
+log = logging.getLogger(__name__)
+
 JPEG_HEAD = b"\xff\xd8\xff"
 JPEG_TAIL = b"\xff\xd9"
 
@@ -108,3 +114,52 @@ def validate_file(path: Path) -> ValidationResult:
         return ValidationResult(ok=True, format="webp", size=size)
 
     return ValidationResult(ok=True, format=None, size=size)
+
+
+def quarantine_file(
+    images_root: Path,
+    path: Path,
+    artist_slug: str,
+    platform: str,
+    *,
+    url: str | None,
+    result: ValidationResult,
+) -> Path | None:
+    """Move a validation-failed file to `_quarantine/<slug>/<platform>` and write
+    a `.quarantine.json` provenance sidecar next to it.
+
+    Returns the destination path, or None if the move itself failed (file left
+    in place — the caller decides what to report). The caller has already run
+    `validate_file` and seen `result.ok is False`. ONE implementation for both
+    download backends — gallery-dl's batch post-process and the native ingester's
+    per-media path — so the quarantine layout + provenance sidecar can't drift.
+    """
+    quarantine_root = images_root / "_quarantine" / artist_slug / platform
+    try:
+        quarantine_root.mkdir(parents=True, exist_ok=True)
+        dest = quarantine_root / path.name
+        counter = 1
+        while dest.exists():
+            dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
+            counter += 1
+        shutil.move(str(path), str(dest))
+        sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
+        sidecar.write_text(
+            json.dumps(
+                {
+                    "original_path": str(path),
+                    "source_url": url,
+                    "artist_slug": artist_slug,
+                    "platform": platform,
+                    "format": result.format,
+                    "reason": result.reason,
+                    "size": result.size,
+                    "quarantined_at": datetime.now(UTC).isoformat(),
+                },
+                indent=2,
+            )
+        )
+    except OSError as exc:
+        log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
+        return None
+    return dest

backend/app/services/gallery_dl.py

@@ -22,7 +22,7 @@ from datetime import UTC, datetime
 from enum import StrEnum
 from pathlib import Path
 
-from .file_validator import is_validatable, validate_file
+from .file_validator import is_validatable, quarantine_file, validate_file
 
 log = logging.getLogger(__name__)
 
@@ -39,6 +39,12 @@ class ErrorType(StrEnum):
     HTTP_ERROR = "http_error"
     UNSUPPORTED_URL = "unsupported_url"
     VALIDATION_FAILED = "validation_failed"
+    # Native Patreon ingester only (plan #697): a response parsed as JSON but
+    # didn't match the JSON:API shape the ingester depends on (missing `data`,
+    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
+    # is updating the ingester's field-set/parser, not rotating credentials. The
+    # contract test guards against silently shipping this.
+    API_DRIFT = "api_drift"
     # Run made real progress (downloaded ≥1 file) but did not finish in the
     # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
     # mapping classifies this as "ok" because the next tick continues.
@@ -55,25 +61,24 @@ class ErrorType(StrEnum):
 # 20 contiguous HEADs is still negligible.
 TICK_SKIP_VALUE = "exit:20"
 
-# Backfill mode (operator-triggered deep scan): walk the full history.
-# Source.backfill_runs_remaining > 0 selects this mode; the longer
-# timeout below absorbs creators with thousands of posts.
+# Backfill mode (operator-triggered deep scan): walk the full history,
+# one TIME-BOXED CHUNK per run (plan #693). config_overrides["_backfill_state"]
+# == "running" selects this mode; the cursor checkpoint (plan #689) lets each
+# chunk resume where the last stopped, so the walk advances across chunks
+# until gallery-dl exits cleanly (= reached the bottom).
 #
-# Sits below download_source's Celery soft_time_limit
-# (DOWNLOAD_SOFT_TIME_LIMIT=1350, tasks/download.py) with ~180s of
-# headroom for phase-3 persist. subprocess.run MUST raise TimeoutExpired
-# before Celery raises SoftTimeLimitExceeded — that exception path
-# captures partial stdout/stderr and finalizes the event; the soft-limit
-# path (until the 2026-06-03 fix) did not. Audit history: 1800 guaranteed
-# SIGKILL against the old hard limit (Knuxy #38275); 1170 was then sized
-# "30s shy of the hard limit (1200)" but still EXCEEDED the soft limit
-# (900), so SoftTimeLimitExceeded preempted TimeoutExpired and every
-# backfill stranded empty (Anduo #39912). Raising the Celery soft/hard
-# limits to 1350/1500 (tasks/download.py) is what made 1170 safe.
-# backfill_runs_remaining=3 still gives ~58 minutes of cumulative walk
-# across three runs for prolific creators.
+# The chunk budget is deliberately FAR below download_source's Celery
+# soft_time_limit (DOWNLOAD_SOFT_TIME_LIMIT=1350, tasks/download.py), not
+# "just under" it. Hitting this budget is the NORMAL chunk boundary, not a
+# failure: subprocess.run raises TimeoutExpired (which captures partial
+# stdout/stderr + the last emitted cursor), and download_service reclassifies
+# a chunk that made progress as PARTIAL (status "ok"), not an error. The huge
+# headroom means a stuck file or a slow chunk can never let Celery's
+# SoftTimeLimitExceeded preempt TimeoutExpired (the failure mode behind Knuxy
+# #38275 / Anduo #39912/#40411). Earlier we ran one ~1170s run-to-the-wall
+# per arming; that died as a timeout error every time on large catalogs.
 BACKFILL_SKIP_VALUE = True
-BACKFILL_TIMEOUT_SECONDS = 1170
+BACKFILL_CHUNK_SECONDS = 600
 
 
 # Sits well below download_source's Celery soft_time_limit
@@ -98,6 +103,17 @@ class SourceConfig:
     Source.backfill_runs_remaining column at the download_service layer
     and is passed to _build_config_for_source as `skip_value`. Same for
     the per-run subprocess timeout.
+
+    `resume_cursor` is RUNTIME state, not persisted operator config: the
+    cursor-paged backfill checkpoint (see parse_last_cursor + the
+    download_service backfill lifecycle). It is consumed only by the native
+    Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path
+    was removed at the #697 cutover): on a backfill/recovery run the ingester
+    resumes its newest→oldest walk from that pagination cursor instead of
+    restarting from the top. download_service threads it from
+    config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read
+    it in from_dict (config_overrides also holds operator config, and
+    resume_cursor must only apply in backfill/recovery mode).
     """
     content_types: list[str] = field(default_factory=lambda: ["all"])
     sleep: float | None = None
@@ -106,6 +122,7 @@ class SourceConfig:
     filename_pattern: str | None = None
     save_metadata: bool = True
     timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
+    resume_cursor: str | None = None
 
     @classmethod
     def from_dict(cls, data: dict) -> SourceConfig:
@@ -130,6 +147,17 @@ class DownloadResult:
     files_quarantined: int = 0
     quarantined_paths: list[str] = field(default_factory=list)
     written_paths: list[str] = field(default_factory=list)
+    # Native ingester only: post-only sidecar paths for posts that have NO
+    # downloadable media (pure-text posts), so the importer can still upsert the
+    # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
+    # Importer.upsert_post_record (keyed on external_post_id → updates, never
+    # doubles, the same Post a media import would create).
+    post_record_paths: list[str] = field(default_factory=list)
+    # Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
+    # pairs for already-present media whose ImageRecord.source_filehash should be
+    # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
+    # on the gallery-dl path and outside recapture.
+    relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0
@@ -138,6 +166,51 @@ class DownloadResult:
     duration_seconds: float = 0.0
     started_at: str | None = None
     completed_at: str | None = None
+    # Plan #704 — structured fields the NATIVE ingester populates directly (None
+    # on the gallery-dl path, which keeps the regex-over-stdout route). When set,
+    # phase 3 reads these instead of scraping the text it would otherwise have to
+    # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
+    # backfill checkpoint the ingester knows exactly (no parse_last_cursor).
+    run_stats: dict | None = None
+    cursor: str | None = None
+    posts_processed: int = 0
+    # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
+    # the platform cooldown matches the hint instead of a flat default. None when
+    # unknown (no header, or not a rate-limit failure).
+    retry_after_seconds: float | None = None
+
+
+def extract_errors_warnings(stderr: str) -> str:
+    """Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
+
+    A generic download-log helper (module-level so the native-ingester result
+    path can shape its logs without reaching through a GalleryDLService instance).
+    """
+    if not stderr:
+        return ""
+    kept = [
+        line for line in stderr.splitlines()
+        if "][error]" in line.lower() or "][warning]" in line.lower()
+    ]
+    return "\n".join(kept)
+
+
+def truncate_log(text: str, max_bytes: int = 500_000) -> str:
+    """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
+    if not text:
+        return text
+    encoded = text.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return text
+    half = max_bytes // 2
+    head = encoded[:half].decode("utf-8", errors="ignore")
+    tail = encoded[-half:].decode("utf-8", errors="ignore")
+    head_lines = head.count("\n")
+    tail_lines = tail.count("\n")
+    total_lines = text.count("\n")
+    elided = max(0, total_lines - head_lines - tail_lines)
+    marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
+    return head + marker + tail
 
 
 def _summarize_validation_failures(failures: list[dict]) -> str:
@@ -154,6 +227,40 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
     return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
 
 
+# (parse_last_cursor was removed in plan #704: the native ingester now carries
+# its checkpoint cursor as a structured DownloadResult.cursor field, so there is
+# no log text to scrape — and gallery-dl platforms never had a cursor.)
+
+
+def make_run_stats(
+    *,
+    exit_code: int = 0,
+    downloaded_count: int = 0,
+    skipped_count: int = 0,
+    per_item_failures: int = 0,
+    warning_count: int = 0,
+    tier_gated_count: int = 0,
+    quarantined_count: int = 0,
+    dead_lettered_count: int = 0,
+) -> dict:
+    """The canonical `run_stats` dict shape, in ONE place.
+
+    Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
+    native ingester's per-outcome tally (ingest_core) — build through this so the
+    key set can't drift between backends. Phase 3 + the Logs UI read these keys.
+    """
+    return {
+        "exit_code": exit_code,
+        "downloaded_count": downloaded_count,
+        "skipped_count": skipped_count,
+        "per_item_failures": per_item_failures,
+        "warning_count": warning_count,
+        "tier_gated_count": tier_gated_count,
+        "quarantined_count": quarantined_count,
+        "dead_lettered_count": dead_lettered_count,
+    }
+
+
 class GalleryDLService:
     """Service for executing gallery-dl downloads."""
 
@@ -187,21 +294,12 @@ class GalleryDLService:
         "permission denied", "tier required", "pledge required",
     ]
 
-    # Per-platform defaults. Lifted from GS — same six platforms FC supports.
+    # Per-platform defaults for the gallery-dl-backed platforms. Patreon was
+    # removed at the plan-#697 cutover — it now uses the native ingester
+    # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
-        "patreon": {
-            "content_types": ["images", "image_large", "attachments", "postfile", "content"],
-            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
-            "filename": "{num:>02}_{filename}.{extension}",
-            "videos": True,
-            "embeds": True,
-            "cursor": True,
-        },
-        "subscribestar": {
-            "content_types": ["all"],
-            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
-            "filename": "{num:>02}_{filename}.{extension}",
-        },
+        # subscribestar removed — it's a native-ingester platform now (#71); the
+        # remaining entries are the gallery-dl platforms not yet migrated.
         "hentaifoundry": {
             "content_types": ["all"],
             "directory": [],
@@ -255,7 +353,12 @@ class GalleryDLService:
                 "skip": True,
                 "sleep": self._rate_limit,
                 "sleep-request": max(0.5, self._rate_limit / 4),
-                "retries": 3,
+                # 2 (not 3) retries — a stuck CDN host shouldn't burn a whole
+                # backfill chunk on one file. Anduo #40838 spent ~600s on a
+                # single image (4 extractor HEAD retries × 30s + 4 downloader
+                # GET retries × 120s); halving retries + the downloader
+                # timeout below caps a wedged file at ~1-2 min instead.
+                "retries": 2,
                 "timeout": 30.0,
                 "verify": True,
                 "postprocessors": [
@@ -270,27 +373,17 @@ class GalleryDLService:
             "downloader": {
                 "part": True,
                 "part-directory": str(self._config_dir / "temp"),
-                "retries": 3,
-                "timeout": 120.0,
-                # Forward Patreon as Referer/Origin to yt-dlp when it
-                # fetches video manifests. Operator-flagged 2026-06-01
-                # (DaferQ patreon): video posts hosted on Mux carry a JWT
-                # playback restriction that checks Referer/Origin on every
-                # request — not just the token signature. gallery-dl's
-                # HEAD probe to stream.mux.com returns 200 (the token is
-                # valid), but yt-dlp's actual GET-with-Range to fetch the
-                # m3u8 manifest 403s because yt-dlp sends its own default
-                # Referer, which Mux's policy rejects. Forcing the right
-                # headers fixes the headers-only case; Mux IP-range
-                # restrictions are unfixable from here.
-                "ytdl": {
-                    "raw-options": {
-                        "http_headers": {
-                            "Referer": "https://www.patreon.com/",
-                            "Origin": "https://www.patreon.com",
-                        },
-                    },
-                },
+                # See the extractor retries note above (Anduo #40838): 2
+                # retries + a 60s read-timeout (was 120) so a stalled
+                # connection fails fast. 60s is a per-read timeout, not a
+                # transfer cap — large GIFs that keep streaming are unaffected.
+                "retries": 2,
+                "timeout": 60.0,
+                # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived
+                # here until the plan-#697 cutover. It was Patreon-specific (and
+                # would have wrongly tagged the other platforms' yt-dlp fetches);
+                # Patreon video is now handled by the native ingester's
+                # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone.
             },
             "output": {"progress": True},
         }
@@ -342,14 +435,7 @@ class GalleryDLService:
 
         platform_section = config["extractor"].setdefault(platform, {})
 
-        if platform == "patreon":
-            if "all" in source_config.content_types:
-                platform_section["files"] = [
-                    "images", "image_large", "attachments", "postfile", "content",
-                ]
-            else:
-                platform_section["files"] = source_config.content_types
-        elif platform == "hentaifoundry":
+        if platform == "hentaifoundry":
             if "pictures" in source_config.content_types or "all" in source_config.content_types:
                 platform_section["include"] = "all"
 
@@ -509,10 +595,6 @@ class GalleryDLService:
         if not written_paths:
             return quarantined_relpaths, failures
 
-        quarantine_root = (
-            self.images_root / "_quarantine" / artist_slug / platform
-        )
-
         for path in written_paths:
             if not is_validatable(path):
                 continue
@@ -523,34 +605,14 @@ class GalleryDLService:
                 continue
             if result.ok:
                 continue
-            try:
-                quarantine_root.mkdir(parents=True, exist_ok=True)
-                dest = quarantine_root / path.name
-                counter = 1
-                while dest.exists():
-                    dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
-                    counter += 1
-                path.rename(dest)
-                sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
-                sidecar.write_text(
-                    json.dumps(
-                        {
-                            "original_path": str(path),
-                            "source_url": url,
-                            "artist_slug": artist_slug,
-                            "platform": platform,
-                            "format": result.format,
-                            "reason": result.reason,
-                            "size": result.size,
-                            "quarantined_at": datetime.now(UTC).isoformat(),
-                        },
-                        indent=2,
-                    )
-                )
-            except OSError as exc:
-                log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
-                continue
-
+            # Shared move+sidecar (file_validator.quarantine_file) — same impl the
+            # native ingester uses, so the layout + provenance sidecar can't drift.
+            dest = quarantine_file(
+                self.images_root, path, artist_slug, platform,
+                url=url, result=result,
+            )
+            if dest is None:
+                continue  # move failed → left in place, not counted
             log.warning(
                 "Quarantined corrupt file: %s → %s (%s: %s)",
                 path, dest, result.format, result.reason,
@@ -588,41 +650,24 @@ class GalleryDLService:
             if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
         )
 
-        return {
-            "exit_code": return_code,
-            "downloaded_count": self._count_downloaded_files(stdout),
-            "skipped_count": skipped_stdout + skipped_stderr,
-            "per_item_failures": per_item_failures,
-            "warning_count": warning_count,
-            "tier_gated_count": tier_gated_count,
-        }
+        return make_run_stats(
+            exit_code=return_code,
+            downloaded_count=self._count_downloaded_files(stdout),
+            skipped_count=skipped_stdout + skipped_stderr,
+            per_item_failures=per_item_failures,
+            warning_count=warning_count,
+            tier_gated_count=tier_gated_count,
+        )
 
     @staticmethod
     def _extract_errors_warnings(stderr: str) -> str:
-        if not stderr:
-            return ""
-        kept = [
-            line for line in stderr.splitlines()
-            if "][error]" in line.lower() or "][warning]" in line.lower()
-        ]
-        return "\n".join(kept)
+        # Thin delegator to the module-level helper (kept for existing callers).
+        return extract_errors_warnings(stderr)
 
     @staticmethod
     def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
-        if not text:
-            return text
-        encoded = text.encode("utf-8")
-        if len(encoded) <= max_bytes:
-            return text
-        half = max_bytes // 2
-        head = encoded[:half].decode("utf-8", errors="ignore")
-        tail = encoded[-half:].decode("utf-8", errors="ignore")
-        head_lines = head.count("\n")
-        tail_lines = tail.count("\n")
-        total_lines = text.count("\n")
-        elided = max(0, total_lines - head_lines - tail_lines)
-        marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
-        return head + marker + tail
+        # Thin delegator to the module-level helper (kept for existing callers).
+        return truncate_log(text, max_bytes)
 
     async def download(
         self,

backend/app/services/gallery_service.py

@@ -14,9 +14,9 @@ Decoding rejects malformed cursors with a ValueError; the API layer
 translates that to HTTP 400.
 """
 
-import base64
 from dataclasses import dataclass
 from datetime import datetime
+from urllib.parse import quote
 
 from sqlalchemy import Select, and_, distinct, exists, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -24,8 +24,14 @@ from sqlalchemy.orm import aliased
 
 from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
 from ..models.tag import image_tag
-
-CURSOR_SEPARATOR = "|"
+from .pagination import decode_cursor, encode_cursor
+from .tag_query import (
+    fandom_join_alias,
+    image_in_any_tag_scope,
+    image_in_tag_scope,
+    serialize_tag,
+    tag_columns,
+)
 
 # Reserved `platform` filter value selecting images with NO platformed
 # provenance (filesystem imports). Returned by facets() as a null-valued
@@ -35,20 +41,6 @@ CURSOR_SEPARATOR = "|"
 UNSOURCED_PLATFORM = "__unsourced__"
 
 
-def encode_cursor(effective_date: datetime, image_id: int) -> str:
-    raw = f"{effective_date.isoformat()}{CURSOR_SEPARATOR}{image_id}"
-    return base64.urlsafe_b64encode(raw.encode()).decode()
-
-
-def decode_cursor(cursor: str) -> tuple[datetime, int]:
-    try:
-        raw = base64.urlsafe_b64decode(cursor.encode()).decode()
-        ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
-        return datetime.fromisoformat(ts_part), int(id_part)
-    except Exception as exc:
-        raise ValueError(f"invalid cursor: {cursor!r}") from exc
-
-
 def _effective_date_col():
     """The materialized gallery sort key: image_record.effective_date
     (alembic 0035) = COALESCE(primary post's post_date, created_at),
@@ -135,11 +127,30 @@ def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str
     return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
 
 
-def _require_single_filter(tag_ids, post_id, artist_id) -> None:
+def image_url(path: str) -> str:
+    """Return the URL to fetch the full-size original from /images.
+
+    The on-disk `path` mirrors the source tree (artist/post folders), so it can
+    contain characters that are special in a URL — most importantly '#' (post
+    titles like 'BLUE#59'), but also spaces, '?', '%'. The serve_image route
+    URL-decodes its <path:subpath> fine, but only if the browser sends the whole
+    path; an unencoded '#' is parsed as a fragment, so '#59/01_timelapse.jpg'
+    never reaches the server and the original 404s while the (hash-named)
+    thumbnail still loads. Percent-encode the path, keeping '/' as the segment
+    separator. Operator-flagged 2026-06-12."""
+    rel = path.split("/images/", 1)[-1]
+    return f"/images/{quote(rel, safe='/')}"
+
+
+def _require_single_filter(
+    tag_ids, post_id, artist_id, tag_or_groups=None, tag_exclude=None,
+) -> None:
     """post_id is the post-detail view — it can't combine with the
-    composable filters. tag_ids + artist_id (+ media_type) compose freely
-    (AND)."""
-    if post_id is not None and (tag_ids or artist_id is not None):
+    composable filters. tag_ids / tag_or_groups / tag_exclude + artist_id
+    (+ media_type) compose freely (AND)."""
+    if post_id is not None and (
+        tag_ids or artist_id is not None or tag_or_groups or tag_exclude
+    ):
         raise ValueError(
             "post_id cannot be combined with tag or artist filters"
         )
@@ -147,6 +158,7 @@ def _require_single_filter(tag_ids, post_id, artist_id) -> None:
 
 def _apply_scope(
     stmt, *, tag_ids, post_id, artist_id, media_type,
+    tag_or_groups=None, tag_exclude=None,
     platform=None, untagged=False, no_artist=False,
     date_from=None, date_to=None,
 ):
@@ -157,7 +169,14 @@ def _apply_scope(
     be present on `stmt` (the artist/platform paths alias Post/Source inside
     their own EXISTS).
 
-    - tag_ids: image must carry ALL of them — one correlated EXISTS per tag.
+    Tag filtering is one structured model (#6): AND-of-OR plus exclusions.
+    - tag_ids: image must carry ALL of them — one correlated EXISTS per tag
+      (the AND-of-singletons "include" common case; light editor + back-compat).
+    - tag_or_groups: list of OR-groups; the image must carry AT LEAST ONE tag
+      from EACH group — one EXISTS(tag_id IN group) per group, AND'd across
+      groups. (advanced editor)
+    - tag_exclude: image must carry NONE of these — a single NOT EXISTS(tag_id
+      IN exclude). (light "exclude" chips + advanced NOT)
     - post_id / artist_id: provenance EXISTS (post_id is exclusive, guarded
       by _require_single_filter).
     - media_type: 'image' | 'video' narrows by mime prefix.
@@ -167,13 +186,17 @@ def _apply_scope(
     - no_artist: ImageRecord.artist_id IS NULL.
     - date_from / date_to: half-open [from, to) bounds on effective_date.
     """
+    # Every tag clause goes through image_in_tag_scope/_any: a fandom tag also
+    # matches images carrying any of its characters (Tag.fandom_id). Include,
+    # OR-group, and exclude are all symmetric on that membership.
     for tid in tag_ids or []:
-        stmt = stmt.where(
-            exists().where(
-                image_tag.c.image_record_id == ImageRecord.id,
-                image_tag.c.tag_id == tid,
-            )
-        )
+        stmt = stmt.where(image_in_tag_scope(tid))
+    for group in tag_or_groups or []:
+        if not group:
+            continue  # an empty OR-group would match nothing; treat as absent
+        stmt = stmt.where(image_in_any_tag_scope(group))
+    if tag_exclude:
+        stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
     prov = _provenance_clause(post_id, artist_id)
     if prov is not None:
         stmt = stmt.where(prov)
@@ -295,6 +318,8 @@ class GalleryService:
         artist_id: int | None = None,
         media_type: str | None = None,
         sort: str = "newest",
+        tag_or_groups: list[list[int]] | None = None,
+        tag_exclude: list[int] | None = None,
         platform: str | None = None,
         untagged: bool = False,
         no_artist: bool = False,
@@ -303,7 +328,9 @@ class GalleryService:
     ) -> GalleryPage:
         if limit < 1 or limit > 200:
             raise ValueError("limit must be between 1 and 200")
-        _require_single_filter(tag_ids, post_id, artist_id)
+        _require_single_filter(
+            tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
+        )
 
         eff = _effective_date_col()
         stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
@@ -311,6 +338,7 @@ class GalleryService:
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
+            tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
             date_from=date_from, date_to=date_to,
         )
@@ -358,6 +386,8 @@ class GalleryService:
         post_id: int | None = None,
         artist_id: int | None = None,
         media_type: str | None = None,
+        tag_or_groups: list[list[int]] | None = None,
+        tag_exclude: list[int] | None = None,
         platform: str | None = None,
         untagged: bool = False,
         no_artist: bool = False,
@@ -371,10 +401,13 @@ class GalleryService:
             year_col, month_col, func.count(ImageRecord.id).label("cnt")
         )
         stmt = _outer_join_primary_post(stmt)
-        _require_single_filter(tag_ids, post_id, artist_id)
+        _require_single_filter(
+            tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
+        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
+            tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
             date_from=date_from, date_to=date_to,
         )
@@ -386,6 +419,8 @@ class GalleryService:
         self, year: int, month: int, tag_ids: list[int] | None = None,
         post_id: int | None = None, artist_id: int | None = None,
         media_type: str | None = None, sort: str = "newest",
+        tag_or_groups: list[list[int]] | None = None,
+        tag_exclude: list[int] | None = None,
         platform: str | None = None, untagged: bool = False,
         no_artist: bool = False, date_from: datetime | None = None,
         date_to: datetime | None = None,
@@ -402,10 +437,13 @@ class GalleryService:
             extract("month", eff) == month,
         )
         stmt = _outer_join_primary_post(stmt)
-        _require_single_filter(tag_ids, post_id, artist_id)
+        _require_single_filter(
+            tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
+        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
+            tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
             date_from=date_from, date_to=date_to,
         )
@@ -426,7 +464,10 @@ class GalleryService:
     async def facets(
         self, *, tag_ids: list[int] | None = None,
         post_id: int | None = None, artist_id: int | None = None,
-        media_type: str | None = None, platform: str | None = None,
+        media_type: str | None = None,
+        tag_or_groups: list[list[int]] | None = None,
+        tag_exclude: list[int] | None = None,
+        platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
     ) -> GalleryFacets:
@@ -436,10 +477,13 @@ class GalleryService:
         No outer join is needed — every clause is a correlated EXISTS or a
         column predicate on ImageRecord.
         """
-        _require_single_filter(tag_ids, post_id, artist_id)
+        _require_single_filter(
+            tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
+        )
         common = {
             "tag_ids": tag_ids, "post_id": post_id,
             "artist_id": artist_id, "media_type": media_type,
+            "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
         }
 
         # total — the full active filter (the headline result count).
@@ -514,7 +558,10 @@ class GalleryService:
     async def similar(
         self, image_id: int, limit: int = 100, *,
         tag_ids: list[int] | None = None, artist_id: int | None = None,
-        media_type: str | None = None, platform: str | None = None,
+        media_type: str | None = None,
+        tag_or_groups: list[list[int]] | None = None,
+        tag_exclude: list[int] | None = None,
+        platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
     ) -> list[GalleryImage] | None:
@@ -546,6 +593,7 @@ class GalleryService:
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=None,
             artist_id=artist_id, media_type=media_type,
+            tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
             date_from=date_from, date_to=date_to,
         )
@@ -558,13 +606,20 @@ class GalleryService:
         record = await self.session.get(ImageRecord, image_id)
         if record is None:
             return None
+        # Self-join Tag to resolve a character's fandom NAME (not just id) so the
+        # modal chip can label it without an N+1 (shared tag_query helpers).
+        fandom_alias = fandom_join_alias()
         tag_stmt = (
-            select(Tag)
-            .join(image_tag, image_tag.c.tag_id == Tag.id)
+            select(*tag_columns(fandom_alias))
+            .select_from(
+                Tag.__table__
+                .join(image_tag, image_tag.c.tag_id == Tag.id)
+                .outerjoin(fandom_alias, Tag.fandom_id == fandom_alias.c.id)
+            )
             .where(image_tag.c.image_record_id == image_id)
             .order_by(Tag.kind.asc(), Tag.name.asc())
         )
-        tags = (await self.session.execute(tag_stmt)).scalars().all()
+        tags = (await self.session.execute(tag_stmt)).all()
         # Fetch the canonical post.post_date for this image (if any) so
         # the modal can show "Posted on <date>" alongside import date.
         posted_at = None
@@ -597,20 +652,12 @@ class GalleryService:
             "created_at": record.created_at.isoformat(),
             "posted_at": posted_at.isoformat() if posted_at else None,
             "thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
-            "image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
+            "image_url": image_url(record.path),
             "artist": (
                 {"id": artist.id, "name": artist.name, "slug": artist.slug}
                 if artist is not None else None
             ),
-            "tags": [
-                {
-                    "id": t.id,
-                    "name": t.name,
-                    "kind": t.kind.value if hasattr(t.kind, "value") else t.kind,
-                    "fandom_id": t.fandom_id,
-                }
-                for t in tags
-            ],
+            "tags": [serialize_tag(t) for t in tags],
             "neighbors": neighbors,
         }
 

backend/app/services/importer.py

@@ -17,12 +17,13 @@ from enum import StrEnum
 from pathlib import Path
 
 from PIL import Image
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from ..models import (
     Artist,
+    ExternalLink,
     ImageProvenance,
     ImageRecord,
     ImportSettings,
@@ -31,12 +32,19 @@ from ..models import (
     Source,
 )
 from ..utils import safe_probe
-from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
+from ..utils.paths import (
+    derive_subdir,
+    derive_top_level_artist,
+    filehash_from_url,
+    hash_suffixed_name,
+    safe_ext,
+)
 from ..utils.phash import compute_phash, find_similar
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
+from .link_extract import extract_external_links
 from .thumbnailer import Thumbnailer
 
 log = logging.getLogger(__name__)
@@ -71,6 +79,21 @@ IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
 VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"}
 ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS
 
+# A "high-resolution files" pack often wraps per-chapter .rar/.zip inside one
+# outer archive (incase #718) — recurse into nested archives so their images
+# land instead of being silently dropped. Cap the depth so a maliciously
+# deep-nested archive can't recurse forever; the bomb-probe re-runs per level.
+_ARCHIVE_MAX_DEPTH = 3
+
+# Tier-1 video near-dup (#871). pHash is images-only, so videos deduped on sha256
+# alone — a different encode/remux of the same clip imported as a distinct record.
+# Identity = container duration within a tight tolerance + matching aspect ratio,
+# scoped to the same artist (codec/bitrate are NOT part of identity — the point is
+# to match ACROSS re-encodes). Quality axis for supersede = pixel dimensions, same
+# as image pHash. Tight tolerances because a wrong video merge is destructive.
+_VIDEO_DUP_DURATION_TOL_SECONDS = 1.0
+_VIDEO_DUP_ASPECT_TOL = 0.02
+
 
 def is_supported(path: Path) -> bool:
     return path.suffix.lower() in ALL_EXTS
@@ -82,27 +105,9 @@ def is_video(path: Path) -> bool:
 
 def _safe_ext(path: Path) -> str:
     """Conservatively extract a file extension for PostAttachment.ext
-    (varchar(32)).
-
-    gallery-dl produces some filenames with URL-encoded query-string
-    artifacts embedded into the basename (e.g.
-    `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
-    `Path.suffix` finds the LAST dot and returns everything after, which
-    in those cases yields a 50+ char "extension" of mostly base64-ish
-    junk. That blows the column. Operator-flagged 2026-05-25.
-
-    Real extensions are short and alphanumeric. We accept anything ≤ 16
-    chars where every post-dot character is alphanumeric; anything else
-    means the input wasn't a real extension and we return the empty
-    string. ext is nullable-ish (empty string still satisfies NOT NULL)
-    and consumers should treat "" as "no known extension".
-    """
-    suffix = path.suffix.lower()
-    if not suffix or len(suffix) > 16:
-        return ""
-    if not all(c.isalnum() for c in suffix[1:]):
-        return ""
-    return suffix
+    (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
+    the Path-typed call sites + the [[path_suffix_sanitize]] memory pointer)."""
+    return safe_ext(path)
 
 
 def _mime_for(path: Path) -> str:
@@ -159,6 +164,14 @@ class Importer:
         self.thumbnailer = thumbnailer
         self.settings = settings
         self.deep = deep
+        # Post-first ingest (#856, milestone #67): on the NATIVE core ingester the
+        # post-record (`upsert_post_record`) is the SOLE writer of a post's
+        # body/links/metadata; the per-media import only links image provenance +
+        # localization, NOT the post body. download_service sets this True for
+        # native platforms before the phase-3 media loop. Default False keeps the
+        # gallery-dl path writing post fields via `_apply_sidecar` (unchanged).
+        # Safe as a per-instance flag: Importer is constructed per Celery task.
+        self.post_first = False
         self.attachments = AttachmentStore(images_root)
         # phash near-dup candidate cache. Archive imports call _import_media
         # per-member; without this cache the per-member SELECT *FROM
@@ -318,6 +331,32 @@ class Importer:
             ),
         )
 
+    def _sync_external_links(self, post: Post) -> None:
+        """Record off-platform file-host links (mega/gdrive/mediafire/dropbox/
+        pixeldrain) found in the post body, so they're never silently dropped
+        and the download worker can fetch them later. Shared by every platform's
+        import (runs off Post.description). INSERT-MISSING only — an existing row
+        keeps its status/attempts (a re-import must not reset a link already
+        downloaded or dead-lettered). Identity is (post_id, url); the full url
+        incl. #fragment is preserved by the extractor."""
+        links = extract_external_links(post.description)
+        if not links:
+            return
+        self.session.flush()  # ensure post.id is assigned before we reference it
+        existing = set(self.session.execute(
+            select(ExternalLink.url).where(ExternalLink.post_id == post.id)
+        ).scalars().all())
+        for link in links:
+            if link.url in existing:
+                continue
+            self.session.add(ExternalLink(
+                post_id=post.id,
+                artist_id=post.artist_id,
+                host=link.host,
+                url=link.url,
+                label=link.label,
+            ))
+
     def import_one(self, source: Path) -> ImportResult:
         """Dispatch by kind. Media → normal pipeline. Archive → extract
         media members (one Post via the archive-adjacent sidecar) and
@@ -374,22 +413,39 @@ class Importer:
             artist = self._resolve_artist(source)
             post = self._post_for_sidecar(source, artist)
         sha = _sha256_of(source)
-        select_existing = select(PostAttachment).where(PostAttachment.sha256 == sha)
+        post_id = post.id if post else None
+        # Dedup is PER-POST, not global: a non-art file the creator attaches to
+        # many posts (a standard pdf/zip/link-card) must get a row on EVERY post
+        # so none is left a bare shell. The on-disk blob stays sha-deduped
+        # (attachments.store is sha-addressed + idempotent); only the rows are
+        # per-post. PostAttachment's unique is partial (post_id, sha256) for
+        # real posts and (sha256) for the NULL-post filesystem case. Before
+        # 2026-06-08 the global UNIQUE(sha256) left every post after the first
+        # with no attachment row → 1589 empty Anduo shells.
+        if post_id is not None:
+            select_existing = select(PostAttachment).where(
+                PostAttachment.post_id == post_id,
+                PostAttachment.sha256 == sha,
+            )
+        else:
+            select_existing = select(PostAttachment).where(
+                PostAttachment.post_id.is_(None),
+                PostAttachment.sha256 == sha,
+            )
         existing = self.session.execute(select_existing).scalar_one_or_none()
         if existing is not None:
             self.session.commit()
             return ImportResult(status="attached")
-        # Savepoint + IntegrityError recovery — PostAttachment.sha256 is
-        # UNIQUE, so two workers can both pass the SELECT and only the
-        # second INSERT fails. Without savepoint, the outer transaction
-        # poisons and the calling task crashes. attachments.store is
-        # sha-addressed so both workers race to write the same target
-        # path; shutil.copy2 + rename is idempotent. Audit 2026-06-02.
+        # Savepoint + IntegrityError recovery — the partial UNIQUE means two
+        # workers can both pass the SELECT and only the second INSERT fails.
+        # Without savepoint, the outer transaction poisons and the calling task
+        # crashes. attachments.store is sha-addressed so both workers race to
+        # write the same target path; shutil.copy2 + rename is idempotent.
         sp = self.session.begin_nested()
         try:
             stored = self.attachments.store(source, sha)
             self.session.add(PostAttachment(
-                post_id=post.id if post else None,
+                post_id=post_id,
                 artist_id=artist.id if artist else None,
                 sha256=sha,
                 path=stored,
@@ -402,7 +458,7 @@ class Importer:
             sp.commit()
         except IntegrityError:
             sp.rollback()
-            # Lost the race — the other worker's row is canonical.
+            # Lost the race — the other worker's row for this (post, sha) wins.
             self.session.execute(select_existing).scalar_one()
         self.session.commit()
         return ImportResult(status="attached")
@@ -443,30 +499,218 @@ class Importer:
             self._capture_attachment(
                 source, post=post, artist=artist_use, resolved=True,
             )
-            return ImportResult(status="attached")
+            reason = f"archive probe rejected, captured unextracted: {probe.reason}"
+            log.warning("%s: %s", source.name, reason)
+            return ImportResult(status="attached", error=reason)
 
         artist_use = artist if artist is not None else self._resolve_artist(source)
         post = self._post_for_sidecar(source, artist_use)
         member_ids: list[int] = []
-        with extract_archive(source) as members:
-            for _name, member_path in members:
-                if not is_supported(member_path):
-                    continue  # non-media preserved via the stored archive
-                res = self._import_media(
-                    member_path, source, explicit_source=source_row,
-                )
-                if res.status in ("imported", "superseded") and res.image_id:
-                    member_ids.append(res.image_id)
+        # Every member image touched (new + superseded + deduped), so the
+        # from_attachment_id stamp below covers files that already existed in the
+        # library and were merely re-linked to this post — those matter most
+        # (the HR copy a bundle re-ships). Separate from member_ids, which is
+        # the NEWLY-imported subset feeding the ImportResult contract.
+        member_record_ids: set[int] = set()
+        # Per-outcome tally so the "no images" reason names the ACTUAL cause
+        # (#718): nested-archive packs, all-deduped (benign), unsupported formats,
+        # or failed/corrupt members — instead of one catch-all string.
+        counts = {
+            "media": 0, "deduped": 0, "unsupported": 0,
+            "failed": 0, "nested": 0, "nested_rejected": 0,
+        }
+        self._collect_archive_members(
+            source, attribution=source, source_row=source_row,
+            depth=0, member_ids=member_ids, counts=counts,
+            member_record_ids=member_record_ids,
+        )
         # Preserve the archive itself (links to the same Post/Artist).
         self._capture_attachment(
             source, post=post, artist=artist_use, resolved=True
         )
+        # Stamp each member's provenance row for THIS post with the archive it
+        # came out of (milestone #87). Done as a post-pass rather than threaded
+        # through _import_media/_apply_sidecar so the many dedup/supersede
+        # branches stay untouched. NULL-only so a re-extract never re-stamps and
+        # the backfill (reextract task → this same path) is idempotent. Nested
+        # members link to this OUTER archive — the only one stored as a blob.
+        self._stamp_member_archive(
+            post.id if post is not None else None, source, member_record_ids,
+        )
         if member_ids:
             return ImportResult(
                 status="imported", image_id=member_ids[0],
                 member_image_ids=member_ids,
             )
-        return ImportResult(status="attached")
+        # No NEW images landed. Name WHY precisely so a post showing "no images"
+        # beside an archive is diagnosable — and DON'T flag the benign all-deduped
+        # case as a problem: those images already exist in the library and were
+        # re-linked to this post (enrich-on-duplicate), so the post DOES show them.
+        if counts["media"] == 0 and counts["nested"] == 0:
+            reason = ("archive yielded no members (unsupported/corrupt, or the "
+                      "extractor backend failed)")
+        elif counts["deduped"] and not (
+            counts["unsupported"] or counts["failed"] or counts["nested_rejected"]
+        ):
+            log.info(
+                "%s: all %d image member(s) already in library (deduped + linked "
+                "to this post) — archive preserved, no new import",
+                source.name, counts["deduped"],
+            )
+            return ImportResult(status="attached")
+        else:
+            reason = (
+                f"no new images — media={counts['media']} deduped={counts['deduped']} "
+                f"unsupported/non-media={counts['unsupported']} failed={counts['failed']} "
+                f"nested={counts['nested']} nested-rejected={counts['nested_rejected']}"
+            )
+        log.warning("%s: %s", source.name, reason)
+        return ImportResult(status="attached", error=reason)
+
+    def _collect_archive_members(
+        self, archive_path: Path, *, attribution: Path,
+        source_row: Source | None, depth: int,
+        member_ids: list[int], counts: dict,
+        member_record_ids: set[int],
+    ) -> None:
+        """Extract `archive_path` and import its image/video members, RECURSING
+        into nested archives (#718). Members attribute to `attribution` — the
+        OUTER archive's path, so its sidecar resolves them to the right Post even
+        when they came from a nested archive. Depth-capped + bomb-probed per
+        nested level. Mutates `member_ids` and `counts` in place.
+
+        extract_archive is fail-soft, but each level is wrapped so one bad nested
+        archive can't abort the whole import."""
+        try:
+            with extract_archive(archive_path) as members:
+                for _name, member_path in members:
+                    if not member_path.is_file():
+                        continue  # directory entries etc. — not media
+                    if is_archive(member_path):
+                        counts["nested"] += 1
+                        if depth + 1 > _ARCHIVE_MAX_DEPTH:
+                            counts["nested_rejected"] += 1
+                            log.warning(
+                                "nested archive past depth cap %d, skipped: %s",
+                                _ARCHIVE_MAX_DEPTH, member_path.name,
+                            )
+                            continue
+                        probe = safe_probe.probe_archive(member_path)
+                        if not probe.ok:
+                            counts["nested_rejected"] += 1
+                            log.warning(
+                                "nested archive rejected (%s): %s",
+                                probe.reason, member_path.name,
+                            )
+                            continue
+                        self._collect_archive_members(
+                            member_path, attribution=attribution,
+                            source_row=source_row, depth=depth + 1,
+                            member_ids=member_ids, counts=counts,
+                            member_record_ids=member_record_ids,
+                        )
+                        continue
+                    counts["media"] += 1
+                    if not is_supported(member_path):
+                        counts["unsupported"] += 1
+                        continue  # non-media preserved via the stored archive
+                    res = self._import_media(
+                        member_path, attribution, explicit_source=source_row,
+                    )
+                    if res.status in ("imported", "superseded") and res.image_id:
+                        member_ids.append(res.image_id)
+                        member_record_ids.add(res.image_id)
+                    elif res.status == "skipped" and res.skip_reason in (
+                        SkipReason.duplicate_hash, SkipReason.duplicate_phash
+                    ):
+                        counts["deduped"] += 1
+                        # A deduped member still links provenance to this post
+                        # (enrich-on-duplicate); record it so its archive origin
+                        # gets stamped too.
+                        if res.image_id:
+                            member_record_ids.add(res.image_id)
+                    else:
+                        counts["failed"] += 1
+        except Exception as exc:  # noqa: BLE001 — defensive per level; keep going
+            log.warning(
+                "archive extraction failed for %s (depth %d): %s",
+                archive_path.name, depth, exc,
+            )
+
+    def _stamp_member_archive(
+        self, post_id: int | None, archive_source: Path, member_record_ids: set[int],
+    ) -> None:
+        """Record which archive each extracted member came from (milestone #87).
+
+        Resolves the archive's own PostAttachment (by post + sha — it was just
+        captured) and stamps from_attachment_id on every member's provenance row
+        FOR THIS POST. NULL-only, so re-extracting the same archive (the backfill
+        path) never overwrites and stays idempotent. No-op when the archive isn't
+        post-attached (filesystem import with no post) or yielded no members.
+        """
+        if post_id is None or not member_record_ids:
+            return
+        sha = _sha256_of(archive_source)
+        att_id = self.session.execute(
+            select(PostAttachment.id).where(
+                PostAttachment.post_id == post_id,
+                PostAttachment.sha256 == sha,
+            )
+        ).scalar_one_or_none()
+        if att_id is None:
+            return
+        self.session.execute(
+            update(ImageProvenance)
+            .where(
+                ImageProvenance.image_record_id.in_(member_record_ids),
+                ImageProvenance.post_id == post_id,
+                ImageProvenance.from_attachment_id.is_(None),
+            )
+            .values(from_attachment_id=att_id)
+        )
+        self.session.commit()
+
+    @staticmethod
+    def _video_aspect_matches(w, h, cw, ch) -> bool:
+        """True when two (w,h) pairs share an aspect ratio within tolerance.
+        Missing dims → don't block (duration is the primary signal)."""
+        if not (w and h and cw and ch):
+            return True
+        return abs((w / h) - (cw / ch)) <= _VIDEO_DUP_ASPECT_TOL
+
+    def _find_similar_video(
+        self, duration: float | None, width: int | None, height: int | None,
+        artist_id: int | None,
+    ) -> tuple[str, int | None]:
+        """Tier-1 video near-dup (#871). Find a same-artist video whose duration
+        matches within tolerance AND whose aspect ratio matches — treat it as the
+        same content (a different encode/remux). Returns find_similar's contract:
+        ("none"|"larger_exists"|"smaller_exists", id), quality judged by pixel
+        dimensions so a higher-res copy supersedes a smaller one."""
+        if duration is None or artist_id is None:
+            return ("none", None)
+        lo = duration - _VIDEO_DUP_DURATION_TOL_SECONDS
+        hi = duration + _VIDEO_DUP_DURATION_TOL_SECONDS
+        rows = self.session.execute(
+            select(ImageRecord.id, ImageRecord.width, ImageRecord.height)
+            .where(
+                ImageRecord.artist_id == artist_id,
+                ImageRecord.mime.like("video/%"),
+                ImageRecord.duration_seconds.is_not(None),
+                ImageRecord.duration_seconds >= lo,
+                ImageRecord.duration_seconds <= hi,
+            )
+        ).all()
+        w, h = width or 0, height or 0
+        for cid, cw, ch in rows:
+            if not self._video_aspect_matches(width, height, cw, ch):
+                continue
+            cw0, ch0 = cw or 0, ch or 0
+            if cw0 >= w and ch0 >= h:
+                return ("larger_exists", cid)
+            if w > cw0 or h > ch0:
+                return ("smaller_exists", cid)
+        return ("none", None)
 
     def _import_media(
         self, source: Path, attribution_path: Path,
@@ -486,13 +730,13 @@ class Importer:
 
         # Compute file dimensions (images only) and apply filters.
         width = height = None
+        duration = None  # video container duration (Tier-1 near-dup key, #871)
         has_alpha = False
         if is_video(source):
             # Layer-3 isolation: validate the container via ffprobe (a
             # separate process) before the rest of the pipeline touches
             # it. A corrupt video that would crash a decoder is rejected
-            # cleanly here, and we capture width/height for free (the
-            # importer didn't previously record video dimensions).
+            # cleanly here, and we capture width/height + duration for free.
             probe = safe_probe.probe_video(source)
             if not probe.ok:
                 if probe.crashed:
@@ -504,7 +748,7 @@ class Importer:
                     status="skipped", skip_reason=SkipReason.invalid_image,
                     error=probe.reason,
                 )
-            width, height = probe.width, probe.height
+            width, height, duration = probe.width, probe.height, probe.duration
         else:
             try:
                 with Image.open(source) as im:
@@ -546,12 +790,30 @@ class Importer:
                         error=f"{pct:.2%} transparent",
                     )
 
+        # Artist anchored to the attribution path (folder→artist), resolved
+        # UP-FRONT so the enrich-on-duplicate branches link provenance with the
+        # right artist even when the sidecar carries none — which is now the norm
+        # for archive members under post-first (the per-media sidecar is minimal).
+        # Without this a cross-posted / re-packed archive image deduped and was
+        # left UNLINKED from the new post = a post showing "no images" (#718).
+        _artist_name = derive_top_level_artist(attribution_path, self.import_root)
+        path_artist = self._upsert_artist(_artist_name) if _artist_name else None
+
         # Hash dedup (exact).
         sha = _sha256_of(source)
         existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
         if existing:
             if not self.deep:
+                # Enrich-on-duplicate (parity with attach_in_place): a re-scanned
+                # file that is a byte-dup of an existing image still links its
+                # post via _apply_sidecar, so a cross-posted image shows on every
+                # post.
+                self._apply_sidecar(
+                    existing, attribution_path, path_artist,
+                    explicit_source=explicit_source,
+                )
+                self.session.commit()
                 return ImportResult(
                     status="skipped", skip_reason=SkipReason.duplicate_hash,
                     image_id=existing.id, error="sha256 already present",
@@ -578,6 +840,14 @@ class Importer:
                     candidates, self.settings.phash_threshold,
                 )
                 if rel == "larger_exists":
+                    # Enrich-on-duplicate (parity with attach_in_place).
+                    larger = self.session.get(ImageRecord, match_id)
+                    if larger is not None:
+                        self._apply_sidecar(
+                            larger, attribution_path, path_artist,
+                            explicit_source=explicit_source,
+                        )
+                        self.session.commit()
                     return ImportResult(
                         status="skipped",
                         skip_reason=SkipReason.duplicate_phash,
@@ -590,6 +860,32 @@ class Importer:
                     return ImportResult(
                         status="superseded", image_id=match_id
                     )
+        elif duration is not None:
+            # Tier-1 video near-dup (#871): same-artist, matching duration+aspect
+            # → same content across re-encodes. Mirror the image flow.
+            rel, match_id = self._find_similar_video(
+                duration, width, height,
+                path_artist.id if path_artist else None,
+            )
+            if rel == "larger_exists":
+                larger = self.session.get(ImageRecord, match_id)
+                if larger is not None:
+                    self._apply_sidecar(
+                        larger, attribution_path, path_artist,
+                        explicit_source=explicit_source,
+                    )
+                    self.session.commit()
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_phash,
+                    image_id=match_id,
+                    error="video near-duplicate (duration+aspect) of existing equal/larger video",
+                )
+            if rel == "smaller_exists":
+                target = self.session.get(ImageRecord, match_id)
+                self._supersede(
+                    target, source, sha, None, width, height, duration=duration,
+                )
+                return ImportResult(status="superseded", image_id=match_id)
 
         dest = self._copy_to_library(source, sha, attribution_path)
 
@@ -601,6 +897,7 @@ class Importer:
             mime=_mime_for(source),
             width=width,
             height=height,
+            duration_seconds=duration,
             origin="imported_filesystem",
             integrity_status="unknown",
         )
@@ -608,13 +905,11 @@ class Importer:
         self.session.flush()
         self._phash_cache_append(phash, width, height, record.id)
 
-        # Folder→artist (anchored to attribution_path).
-        artist = None
-        artist_name = derive_top_level_artist(
-            attribution_path, self.import_root
-        )
-        if artist_name:
-            artist = self._attach_artist(record, artist_name)
+        # Folder→artist (path_artist resolved up-front above); bind it to the
+        # new record so it carries the canonical Artist.
+        if path_artist is not None and record.artist_id is None:
+            record.artist_id = path_artist.id
+            self.session.flush()
 
         # Sidecar provenance (best-effort; never fails the import).
         # explicit_source lets the FC-3c download path bind the new
@@ -624,7 +919,7 @@ class Importer:
         # subscription-downloaded zip previously lost subscription
         # linkage if the on-disk layout didn't match assumptions.
         self._apply_sidecar(
-            record, attribution_path, artist, explicit_source=explicit_source,
+            record, attribution_path, path_artist, explicit_source=explicit_source,
         )
 
         # Thumbnail is queued separately by the calling task; the importer
@@ -673,6 +968,79 @@ class Importer:
         self.session.commit()
         return ImportResult(status="refreshed", image_id=existing.id)
 
+    def _apply_post_fields(self, post: Post, sd) -> None:
+        """Write a parsed sidecar's post-level fields onto a Post — the SINGLE
+        predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
+        and the post-record path (upsert_post_record). Fill-with-non-empty:
+        parse_sidecar yields None for empty values, so a None field is left
+        untouched (an empty feed body never wipes a populated one). raw_metadata +
+        external-link sync always run (latest snapshot). Keeping ONE copy stops
+        the two paths from diverging on how a post body/links get stored — they
+        were verbatim duplicates (#842 DRY pass; see [[feedback_preview_apply_parity]]).
+        """
+        if sd.post_url is not None:
+            post.post_url = sd.post_url
+        if sd.post_title is not None:
+            post.post_title = sd.post_title
+        if sd.post_date is not None:
+            post.post_date = sd.post_date
+        if sd.description is not None:
+            post.description = sd.description
+        if sd.attachment_count is not None:
+            post.attachment_count = sd.attachment_count
+        post.raw_metadata = sd.raw
+        self._sync_external_links(post)
+
+    def upsert_post_record(
+        self, sidecar: Path, *, artist: Artist | None = None,
+        source: Source | None = None,
+    ) -> bool:
+        """Upsert the Post for a post-ONLY sidecar (a media-less post), so the
+        artist archive includes text posts (their body + external links).
+
+        Reuses `_find_or_create_post` (keyed on external_post_id) so it UPDATES
+        the SAME Post a media import would create — never doubles. Fields are
+        FILLED, never clobbered with empty: parse_sidecar yields None for empty
+        values, and a None field is left untouched (an empty feed body never
+        wipes a populated one). Returns True if a Post was upserted, False if the
+        sidecar was unusable (parse failure / no artist)."""
+        try:
+            data = json.loads(sidecar.read_text("utf-8"))
+            if not isinstance(data, dict):
+                raise ValueError("sidecar JSON is not an object")
+        except Exception as exc:
+            log.warning("post-record sidecar parse failed for %s: %s", sidecar, exc)
+            return False
+
+        sd = parse_sidecar(data)
+
+        if artist is None:
+            name = self._sidecar_artist_name(data)
+            artist = self._upsert_artist(name) if name else None
+        if artist is None:
+            log.warning("post-record sidecar %s has no artist; skipping", sidecar)
+            return False
+
+        if source is not None:
+            src = source
+        else:
+            platform = sd.platform or "unknown"
+            src = self._lookup_source_for_sidecar(
+                artist_id=artist.id, platform=platform,
+            )
+
+        epid = sd.external_post_id or sidecar.stem
+        post = self._find_or_create_post(
+            source_id=src.id if src else None,
+            external_post_id=epid,
+            artist_id=artist.id,
+        )
+        if post.artist_id is None:
+            post.artist_id = artist.id
+        self._apply_post_fields(post, sd)
+        self.session.commit()
+        return True
+
     def attach_in_place(
         self,
         path: Path,
@@ -723,6 +1091,7 @@ class Importer:
 
         # Format / dimension / transparency filters (mirror _import_media).
         width = height = None
+        duration = None  # video container duration (Tier-1 near-dup key, #871)
         has_alpha = False
         if not is_video(path):
             try:
@@ -754,6 +1123,14 @@ class Importer:
                         status="skipped", skip_reason=SkipReason.too_transparent,
                         error=f"{pct:.2%} transparent",
                     )
+        else:
+            # Best-effort probe for dims + duration so downloaded videos can dedup
+            # (#871). LENIENT: unlike _import_media this path does not reject on a
+            # probe failure — preserve the existing "import the downloaded video"
+            # behavior; we just skip dedup when we can't read the duration.
+            vp = safe_probe.probe_video(path)
+            if vp.ok:
+                width, height, duration = vp.width, vp.height, vp.duration
 
         # Hash dedup
         sha = _sha256_of(path)
@@ -761,6 +1138,15 @@ class Importer:
             select(ImageRecord).where(ImageRecord.sha256 == sha)
         ).scalar_one_or_none()
         if existing:
+            # Enrich-on-duplicate (image_provenance docstring / spec §3): the
+            # same bytes appearing in another post must show on THAT post too,
+            # so append a provenance row for the new post rather than dropping
+            # the linkage. primary_post_id stays on the first post — _apply_sidecar
+            # only sets it when NULL. Without this the new post is left with no
+            # image AND (if its other content also deduped) no attachment, i.e. a
+            # bare shell — operator-flagged 2026-06-08 (1589 empty Anduo posts).
+            self._apply_sidecar(existing, path, artist, explicit_source=source)
+            self.session.commit()
             return ImportResult(
                 status="skipped", skip_reason=SkipReason.duplicate_hash,
                 image_id=existing.id, error="sha256 already present",
@@ -781,6 +1167,15 @@ class Importer:
                     candidates, self.settings.phash_threshold,
                 )
                 if rel == "larger_exists":
+                    # Enrich-on-duplicate: link the near-dup's post to the
+                    # existing larger image so it shows on both (see duplicate_hash
+                    # above). smaller_exists already links via _supersede.
+                    larger = self.session.get(ImageRecord, match_id)
+                    if larger is not None:
+                        self._apply_sidecar(
+                            larger, path, artist, explicit_source=source,
+                        )
+                        self.session.commit()
                     return ImportResult(
                         status="skipped",
                         skip_reason=SkipReason.duplicate_phash,
@@ -794,6 +1189,30 @@ class Importer:
                         new_path=path, artist=artist, source_row=source,
                     )
                     return ImportResult(status="superseded", image_id=match_id)
+        elif duration is not None:
+            # Tier-1 video near-dup (#871): same content re-downloaded from another
+            # source/encode. artist is the subscription artist (passed explicitly).
+            rel, match_id = self._find_similar_video(
+                duration, width, height, artist.id if artist else None,
+            )
+            if rel == "larger_exists":
+                larger = self.session.get(ImageRecord, match_id)
+                if larger is not None:
+                    self._apply_sidecar(larger, path, artist, explicit_source=source)
+                    self.session.commit()
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_phash,
+                    image_id=match_id,
+                    error="video near-duplicate (duration+aspect) of existing equal/larger video",
+                )
+            if rel == "smaller_exists":
+                target = self.session.get(ImageRecord, match_id)
+                self._supersede(
+                    target, path, sha, None, width, height,
+                    new_path=path, artist=artist, source_row=source,
+                    duration=duration,
+                )
+                return ImportResult(status="superseded", image_id=match_id)
 
         # Create record — the path IS where the file lives.
         record = ImageRecord(
@@ -804,6 +1223,7 @@ class Importer:
             mime=_mime_for(path),
             width=width,
             height=height,
+            duration_seconds=duration,
             origin="downloaded",
             integrity_status="unknown",
         )
@@ -832,17 +1252,6 @@ class Importer:
             self.session.flush()
         return artist
 
-    def _attach_artist(self, record: ImageRecord, artist_name: str) -> Artist:
-        """Upsert the Artist and set it as the image's canonical artist.
-        Artist-kind tags were retired in FC-2d-vii-c — the Artist row is
-        the single source of truth. Returns the Artist (sidecar
-        provenance attaches its Source to it)."""
-        artist = self._upsert_artist(artist_name)
-        if record.artist_id is None:
-            record.artist_id = artist.id
-            self.session.flush()
-        return artist
-
     @staticmethod
     def _sidecar_artist_name(data: dict) -> str | None:
         for k in ("artist", "author_name"):
@@ -860,6 +1269,35 @@ class Importer:
                         return sv.strip()
         return None
 
+    def relink_source_filehash(
+        self, path: Path, source_url: str, *, artist: Artist | None = None,
+    ) -> bool:
+        """#830 recapture: backfill source_url + source_filehash on an EXISTING
+        on-disk image's ImageRecord (matched by sha256) so its post-body inline
+        `<img src=CDN>` remaps to the local copy at render time — WITHOUT
+        re-downloading and WITHOUT unlinking the file (unlike the import path's
+        duplicate-hash branch). NULL-only, mirroring _apply_sidecar: never
+        clobbers an already-set filehash. Returns True only when a row was
+        updated. No-op when the file is gone, the url has no filehash, no record
+        matches the bytes, or the record already carries a filehash.
+        """
+        fh = filehash_from_url(source_url)
+        if not fh:
+            return False
+        try:
+            sha = _sha256_of(path)
+        except OSError:
+            return False
+        record = self.session.execute(
+            select(ImageRecord).where(ImageRecord.sha256 == sha)
+        ).scalar_one_or_none()
+        if record is None or record.source_filehash is not None:
+            return False
+        record.source_url = source_url
+        record.source_filehash = fh
+        self.session.commit()
+        return True
+
     def _apply_sidecar(
         self,
         record: ImageRecord,
@@ -902,6 +1340,14 @@ class Importer:
         if record.artist_id is None:
             record.artist_id = artist.id
 
+        # #830 Phase 2: persist the file's CDN source identity (NULL-only, so a
+        # re-import / enrich-on-duplicate never clobbers it) — the filehash is
+        # the join key that lets post_feed_service remap the body's inline
+        # `<img src=CDN>` to this local copy at render time.
+        if sd.source_url and record.source_filehash is None:
+            record.source_url = sd.source_url
+            record.source_filehash = filehash_from_url(sd.source_url)
+
         if explicit_source is not None:
             src = explicit_source
         else:
@@ -916,17 +1362,14 @@ class Importer:
             external_post_id=epid,
             artist_id=artist.id,
         )
-        if sd.post_url is not None:
-            post.post_url = sd.post_url
-        if sd.post_title is not None:
-            post.post_title = sd.post_title
-        if sd.post_date is not None:
-            post.post_date = sd.post_date
-        if sd.description is not None:
-            post.description = sd.description
-        if sd.attachment_count is not None:
-            post.attachment_count = sd.attachment_count
-        post.raw_metadata = sd.raw
+        # Post-first (#856): on the native path the post-record owns the body/
+        # links/metadata, so the per-media import must NOT write post fields — its
+        # minimal sidecar carries no body, and applying it would clobber
+        # raw_metadata + re-run link-sync off empty data. The image still links
+        # provenance + localization below. gallery-dl (post_first False) is
+        # unchanged: its sidecar IS the only body source, so it still applies.
+        if not self.post_first:
+            self._apply_post_fields(post, sd)
 
         # Race-safe (image_record_id, post_id) upsert — mirrors the
         # _find_or_create_source/post savepoint pattern. The plain
@@ -996,6 +1439,7 @@ class Importer:
         *, new_path: Path | None = None,
         artist: Artist | None = None,
         source_row: Source | None = None,
+        duration: float | None = None,
     ) -> None:
         """Replace `existing`'s file with the larger `source`, keeping the
         row id (so tags/series/curation stay attached). ML is cleared so
@@ -1028,13 +1472,23 @@ class Importer:
         existing.mime = _mime_for(source)
         existing.width = width
         existing.height = height
+        existing.duration_seconds = duration  # #871: keep the kept copy's duration
         existing.thumbnail_path = None
         existing.integrity_status = "unknown"
-        existing.tagger_predictions = None
         existing.tagger_model_version = None
         existing.siglip_embedding = None
         existing.siglip_model_version = None
         existing.centroid_scores = None
+        # #768: predictions also live in the normalized image_prediction table
+        # now — clear them so a re-imported file re-derives a fresh set.
+        from sqlalchemy import delete as _delete
+
+        from ..models import ImagePrediction as _ImagePrediction
+        self.session.execute(
+            _delete(_ImagePrediction).where(
+                _ImagePrediction.image_record_id == existing.id
+            )
+        )
         # created_at intentionally preserved; updated_at auto-bumps.
         self.session.flush()
         self.session.commit()

backend/app/services/ingest_core.py

@@ -0,0 +1,854 @@
+"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705).
+
+The orchestration that drives a native subscription walk — page a feed →
+extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download →
+mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped
+`DownloadResult`, across tick/backfill/recovery modes — is identical for every
+platform. Only four things are platform-specific, and they're INJECTED at
+construction by a thin adapter (e.g. `PatreonIngester`):
+
+  - `client`    — `.iter_posts(feed_id, cursor)` yielding `(post, included,
+                  page_cursor)` + `.extract_media(post, included) -> [media]`.
+  - `downloader`— `.download_post(post, media, artist_slug, is_seen,
+                  should_stop) -> [MediaOutcome]` (status in downloaded/
+                  skipped_seen/skipped_disk/quarantined/error;
+                  `.path`/`.error`/`.post_id`). `should_stop()` is polled
+                  between media so the time-box is honoured mid-post.
+  - ledger      — `seen_model` + `failed_model` SQLAlchemy models (+ their
+                  on-conflict UNIQUE constraint names) and a `ledger_key(media)`.
+  - failure map — the adapter overrides `_failure_result` (platform exception
+                  → DownloadResult.error_type) and supplies `error_base` (the
+                  exception type the walk catches) + `platform` (result label).
+
+Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker —
+never held across a network fetch ([[db-connection-held-across-subprocess]]).
+Plain-HTTP homelab: no secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable
+
+from sqlalchemy import delete, func, select, text
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from .gallery_dl import DownloadResult, ErrorType, make_run_stats
+from .native_ingest_common import NativeAuthError, NativeDriftError
+
+log = logging.getLogger(__name__)
+
+# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or
+# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of
+# per-file HEADs. Headroom against paywalled/undownloadable items interleaving.
+_TICK_SEEN_THRESHOLD = 20
+
+# plan #705 #7: after this many failed download/validate attempts a media is
+# "dead-lettered" and skipped on routine tick/backfill walks (recovery still
+# re-attempts it). Stops a permanently-broken media re-erroring forever.
+DEAD_LETTER_THRESHOLD = 3
+# last_error is Text but bound it so a giant traceback doesn't bloat the row.
+_ERROR_MAX = 1000
+
+# plan #709: throttle the live-progress write to the running DownloadEvent to one
+# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a
+# page is (page boundaries can be minutes apart on image-dense backfills, so a
+# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s).
+_LIVE_PROGRESS_INTERVAL = 5.0
+
+# Post-body schema-drift canary (#862). Patreon's body lives in
+# content/content_json_string with NO post_type gate, so a field rename (as
+# content→content_json_string already was) zeroes EVERY body at once — across
+# every artist, every walk. If a native walk records at least this many posts
+# and extracts a body from NONE of them, treat it as that break (fail the run
+# API_DRIFT) rather than silently archiving empties. A *fraction* threshold would
+# false-positive on gallery/art creators who legitimately post images with no
+# caption, so the gate is "zero across a minimum sample": a real creator nearly
+# always has SOME text across this many posts, a broken parser has none. Set high
+# enough that a small tick (a few new posts) can't trip it — only a backfill /
+# recapture (the operator's schema-test flow) reaches the sample.
+_CANARY_MIN_SAMPLE = 30
+
+
+class Ingester:
+    """Generic native-ingest orchestration. Subclass with a platform adapter
+    (see the module docstring) — or construct directly with the keyword seams."""
+
+    def __init__(
+        self,
+        *,
+        client,
+        downloader,
+        session_factory: Callable[[], object],
+        seen_model,
+        failed_model,
+        seen_constraint: str,
+        failed_constraint: str,
+        ledger_key: Callable[[object], str],
+        platform: str,
+        error_base: type[Exception],
+        drift_label: str | None = None,
+    ):
+        self.client = client
+        self.downloader = downloader
+        self.session_factory = session_factory
+        self._seen_model = seen_model
+        self._failed_model = failed_model
+        self._seen_constraint = seen_constraint
+        self._failed_constraint = failed_constraint
+        self._ledger_key = ledger_key
+        self._platform = platform
+        self._error_base = error_base
+        # Human label for the API_DRIFT message ("<label> changed — ingester needs
+        # update"). Defaults to the platform name; adapters pass a richer phrase
+        # (e.g. "Patreon API", "SubscribeStar markup").
+        self._drift_label = drift_label or platform
+
+    # -- public ------------------------------------------------------------
+
+    def run(
+        self,
+        *,
+        source_id: int,
+        campaign_id: str,
+        artist_slug: str,
+        url: str,
+        mode: str,
+        resume_cursor: str | None = None,
+        time_budget_seconds: float = 870.0,
+        seen_threshold: int = _TICK_SEEN_THRESHOLD,
+        posts_base: int = 0,
+        event_id: int | None = None,
+    ) -> DownloadResult:
+        """Walk + download for one source, returning a gallery-dl-shaped result.
+
+        `mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery
+        bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk
+        still skips kept files). Recapture (#830) is the cheap "re-grab post
+        text" walk: it bypasses the post-record gate (so EVERY post's body +
+        external links are re-captured / detail-fetched) but KEEPS the media
+        seen-ledger — on-disk media is NOT re-downloaded, only surfaced so its
+        ImageRecord's source_filehash can be backfilled for inline-image
+        localization. The walk stops on:
+          - budget exhaustion (time_budget_seconds)         → TIMEOUT / PARTIAL
+          - tick early-out (seen_threshold contiguous seen)  → success
+          - reaching the bottom of the feed                  → success (rc 0)
+        A client-level failure (drift / auth / network) fails the whole run loud.
+        """
+        bypass_seen = mode == "recovery"
+        recapture = mode == "recapture"
+        # Both recovery and recapture re-capture EVERY post's body + links — they
+        # bypass the post-record seen-gate (recovery via bypass_seen, recapture
+        # explicitly). A plain backfill stays gated (capture once per post).
+        recapture_records = bypass_seen or recapture
+        # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
+        # tick has no resumable backfill state.
+        checkpoint = mode in ("backfill", "recovery", "recapture")
+        ledger_key = self._ledger_key
+        # POST-FIRST CONTRACT (milestone #67): these two optional seams make a
+        # platform "post-first" on the native core ingester — the post-record is
+        # the single authoritative writer of the post body/links/metadata, and the
+        # per-media sidecar carries image identity only (download_service flips
+        # importer.post_first via uses_native_ingester, so the import side follows
+        # automatically). A platform migrating off gallery-dl onto the native core
+        # adopts post-first by implementing BOTH:
+        #   client.post_record_key(post)   -> (ledger_key, post_id) | None  (gate)
+        #   downloader.write_post_record(post, artist_slug) -> PostRecordOutcome
+        # Absent on stub clients/downloaders (unit tests) and on not-yet-migrated
+        # platforms → media-less posts are skipped as before and the body still
+        # comes from the per-media sidecar (gallery-dl path). See [[post-first-ingest-contract]].
+        post_record_key = getattr(self.client, "post_record_key", None)
+        write_post_record = getattr(self.downloader, "write_post_record", None)
+        # #874: optional client seam — skip tier-gated posts (the account can't
+        # view them, so Patreon serves only blurred locked-preview media) ENTIRELY:
+        # no media download, no post-record stub. Absent on stub/not-yet-migrated
+        # clients → nothing is ever treated as gated.
+        post_is_gated = getattr(self.client, "post_is_gated", None)
+        start = time.monotonic()
+        last_live = start  # plan #709: last live-progress write timestamp
+        log_lines: list[str] = []
+        written: list[str] = []
+        post_records: list[str] = []
+        quarantined_paths: list[str] = []
+        # #830 recapture: (on-disk path, CDN source_url) pairs for already-present
+        # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
+        # re-downloading or unlinking the file. Empty outside recapture mode.
+        relink: list[tuple[str, str]] = []
+        downloaded = 0
+        errors = 0
+        quarantined = 0
+        dead_lettered = 0
+        skipped_count = 0
+        posts_processed = 0
+        # Post-body schema-drift canary counters (#862, native ingester only —
+        # gallery-dl walks never enter the post-record block below so these stay 0
+        # and the canary can't fire there). posts_recorded = post-records attempted
+        # this walk; posts_with_body = how many yielded a non-empty body.
+        posts_recorded = 0
+        posts_with_body = 0
+        # Tier-gated posts skipped entirely this walk (#874) — surfaced in the
+        # run summary for diagnostics ("a lot of these now").
+        gated_skipped = 0
+        # Net-new posts THIS chunk for the live progress badge (plan #704 #5);
+        # excludes the re-walked resume page so _backfill_posts stays a monotonic
+        # absolute across chunks instead of an inflating sum. posts_processed
+        # stays the gross per-chunk count used for the run summary.
+        chunk_new_posts = 0
+        consecutive_seen = 0
+        emitted_cursor: str | None = None
+        reached_bottom = False
+        budget_hit = False
+        early_out = False
+        stopped = False  # plan #708 B4: operator hit Stop mid-walk
+        cancel_armed = False  # latched once we observe a live "running" state
+
+        def _result(
+            *, success: bool, return_code: int,
+            error_type: ErrorType | None, error_message: str | None,
+        ) -> DownloadResult:
+            # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor
+            # directly instead of regex-scraping a reconstructed stdout. stdout
+            # stays a human-readable summary (no fake `Cursor:` lines).
+            return DownloadResult(
+                success=success,
+                url=url,
+                artist_slug=artist_slug,
+                platform=self._platform,
+                files_downloaded=downloaded,
+                files_quarantined=quarantined,
+                quarantined_paths=list(quarantined_paths),
+                written_paths=written,
+                post_record_paths=list(post_records),
+                relink_source_paths=list(relink),
+                stdout="\n".join(log_lines),
+                stderr="",
+                return_code=return_code,
+                error_type=error_type,
+                error_message=error_message,
+                duration_seconds=time.monotonic() - start,
+                cursor=emitted_cursor,
+                posts_processed=posts_processed,
+                run_stats=make_run_stats(
+                    exit_code=return_code,
+                    downloaded_count=downloaded,
+                    skipped_count=skipped_count,
+                    per_item_failures=errors,
+                    quarantined_count=quarantined,
+                    dead_lettered_count=dead_lettered,
+                ),
+            )
+
+        # #899 L1: emit run milestones through the real logger (not only the
+        # in-memory log_lines → DownloadResult.stdout, which is persisted to the
+        # DownloadEvent ONLY at phase 3). A worker SIGKILL/OOM/hard-time-limit
+        # mid-walk would otherwise leave NO trace; these land in the container log
+        # in real time regardless of whether the event gets finalized.
+        log.info(
+            "%s ingest START (%s): source=%s campaign=%s resume_cursor=%s",
+            self._platform, mode, source_id, campaign_id, resume_cursor,
+        )
+        try:
+            for post, included, page_cursor in self.client.iter_posts(
+                campaign_id, cursor=resume_cursor
+            ):
+                # Checkpoint the cursor that FETCHED this page the moment we
+                # START it — so a chunk cut mid-page resumes the page, not the one
+                # after it. Carried as DownloadResult.cursor (plan #704).
+                if page_cursor and page_cursor != emitted_cursor:
+                    emitted_cursor = page_cursor
+                    # #899 L1: a per-page breadcrumb in the container log (pages can
+                    # be minutes apart on image-dense backfills) — survives a worker
+                    # kill so the operator sees how far a since-died walk got.
+                    log.info(
+                        "%s ingest progress (%s, source=%s): posts=%d downloaded=%d "
+                        "skipped=%d errors=%d quarantined=%d gated=%d cursor=%s",
+                        self._platform, mode, source_id, posts_processed, downloaded,
+                        skipped_count, errors, quarantined, gated_skipped, emitted_cursor,
+                    )
+                    # plan #705 #6: persist the cursor at each page boundary so a
+                    # worker SIGKILL mid-chunk resumes near the crash, not the
+                    # chunk start. (phase 3 still writes the final cursor — same
+                    # value; this is the crash-safety net.) plan #704 #5: persist
+                    # the live posts count alongside it so the badge climbs DURING
+                    # the chunk, not only when it ends.
+                    if checkpoint:
+                        # plan #708 B4: an operator Stop pops `_backfill_state` —
+                        # bail at the page boundary (progress already checkpointed)
+                        # before more network work, so the live chunk halts
+                        # promptly instead of running to its time-box. LATCH on the
+                        # first observed "running" state, so a run invoked WITHOUT a
+                        # running state (a unit test, or a stale call) never
+                        # spuriously self-cancels. A short SELECT, never held.
+                        if self._still_running(source_id):
+                            cancel_armed = True
+                        elif cancel_armed:
+                            stopped = True
+                            break
+                        self._checkpoint_cursor(source_id, emitted_cursor)
+                        self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
+
+                # Time-box check at the post boundary (coarse, like a gallery-dl
+                # chunk). Backfill/recovery resume from emitted_cursor next chunk.
+                if time.monotonic() - start >= time_budget_seconds:
+                    budget_hit = True
+                    break
+
+                posts_processed += 1
+                # The resume page (its cursor == resume_cursor) was already
+                # counted by the chunk that checkpointed it — don't re-count it
+                # into the persisted badge (plan #704 #5). First chunk has
+                # resume_cursor None, so everything counts.
+                if not (resume_cursor and page_cursor == resume_cursor):
+                    chunk_new_posts += 1
+                # Tier-gated post (#874): the account can't fully view it, so
+                # Patreon serves only blurred locked-preview media. Skip it
+                # ENTIRELY — no media download AND no post-record stub (operator
+                # decision: gated content leaves no trace; a later walk re-ingests
+                # it for real once access is gained). Skipped BEFORE the
+                # post-record block so gated posts never inflate the #862 body
+                # canary's sample. post_is_gated gates only on an explicit
+                # current_user_can_view=False (missing/None → viewable).
+                if post_is_gated and post_is_gated(post):
+                    gated_skipped += 1
+                    log_lines.append(
+                        f"  post {post.get('id')} — gated (skipped, no access)"
+                    )
+                    continue
+                # Capture the post body + external links ONCE per post (gated by
+                # the synthetic post key in the seen-ledger), for EVERY post —
+                # whether or not it has downloadable media. This is what makes a
+                # backfill/recovery re-walk RECAPTURE bodies + links for posts
+                # whose media is already on disk: re-downloading existing media
+                # never fills links the system never had, so the body recapture
+                # has to ride the walk itself. Detail-fetch (for an empty feed
+                # body) happens at most once per post — the gate then spares it on
+                # later walks. bypass_seen (recovery) re-captures unconditionally.
+                if post_record_key and write_post_record:
+                    rk = post_record_key(post)
+                    if rk is not None:
+                        pkey, ppid = rk
+                        already = (
+                            set() if recapture_records
+                            else self._seen_keys(source_id, [pkey])
+                        )
+                        if pkey not in already:
+                            rec = write_post_record(post, artist_slug)
+                            posts_recorded += 1
+                            if rec.body_chars:
+                                posts_with_body += 1
+                            if rec.path is not None:
+                                post_records.append(str(rec.path))
+                                self._mark_seen(source_id, [(pkey, ppid)])
+                            # Per-post handling line in the run stdout (the existing
+                            # "Raw stdout" panel) — the downloader already read the
+                            # post; we only format its outcome here. post_type beside
+                            # a 0-char body is the "why is this one empty" answer.
+                            log_lines.append(
+                                f"  post {ppid} [{rec.post_type or '?'}] "
+                                f"body: {rec.body_chars} chars"
+                                + ("" if rec.body_chars else " — EMPTY")
+                                + (f" — {rec.title}" if rec.title else "")
+                            )
+
+                media = self.client.extract_media(post, included)
+                if not media:
+                    continue
+
+                keys = [ledger_key(m) for m in media]
+                # Recovery bypasses BOTH the seen-ledger AND the dead-letter
+                # ledger (the operator's "try everything again"); routine walks
+                # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7).
+                dead = set() if bypass_seen else self._dead_keys(source_id, keys)
+                seen = (
+                    set()
+                    if bypass_seen
+                    else self._seen_keys(source_id, keys)
+                )
+                skip = seen | dead
+
+                def _is_skip(m, _skip=skip) -> bool:
+                    return ledger_key(m) in _skip
+
+                # Honour the time-box DURING a media-dense post too, not only at
+                # the per-post boundary below — else one heavy post can blow the
+                # chunk budget out to the Celery soft limit (Pocketacer, 2026-06-07).
+                outcomes = self.downloader.download_post(
+                    post, media, artist_slug, is_seen=_is_skip,
+                    should_stop=lambda: time.monotonic() - start >= time_budget_seconds,
+                    recapture=recapture,
+                )
+
+                to_mark: list[tuple[str, str]] = []
+                to_clear: list[str] = []  # recovered → drop any dead-letter row
+                to_fail: list[tuple[str, str, str]] = []  # (key, post_id, error)
+                for media_item, outcome in zip(media, outcomes, strict=False):
+                    key = ledger_key(media_item)
+                    if key in dead:
+                        dead_lettered += 1  # skipped because previously dead
+                    if outcome.status == "downloaded":
+                        downloaded += 1
+                        if outcome.path is not None:
+                            written.append(str(outcome.path))
+                        to_mark.append((key, media_item.post_id))
+                        to_clear.append(key)
+                        consecutive_seen = 0
+                    elif outcome.status == "skipped_disk":
+                        # Already on disk (a prior run). Reconcile the ledger so a
+                        # later tick skips it at tier-1 without a disk stat, but
+                        # do NOT re-feed it to phase 3 — attach_in_place would see
+                        # the duplicate sha256 and unlink the on-disk copy.
+                        to_mark.append((key, media_item.post_id))
+                        to_clear.append(key)
+                        skipped_count += 1
+                        consecutive_seen += 1
+                        # #830 recapture: surface (on-disk path, CDN url) so phase
+                        # 3 can backfill source_filehash for inline-image
+                        # localization — a SEPARATE non-deleting channel, never the
+                        # import list (which would unlink the file, per above).
+                        if recapture and outcome.path is not None:
+                            relink.append((str(outcome.path), media_item.url))
+                    elif outcome.status == "skipped_seen":
+                        skipped_count += 1
+                        consecutive_seen += 1
+                    elif outcome.status == "quarantined":
+                        # New content that failed validation (corrupt) — counted
+                        # distinctly so the run surfaces a real quarantined total.
+                        # Not marked seen (a later walk may re-fetch a fixed file);
+                        # it IS new content, so it breaks the run-of-seen. Counts
+                        # toward the dead-letter ledger (plan #705 #7).
+                        quarantined += 1
+                        if outcome.path is not None:
+                            quarantined_paths.append(str(outcome.path))
+                        to_fail.append((key, media_item.post_id, outcome.error or "quarantined"))
+                        consecutive_seen = 0
+                    elif outcome.status == "error":
+                        errors += 1
+                        to_fail.append((key, media_item.post_id, outcome.error or "error"))
+                        # An error neither advances nor resets the run-of-seen.
+
+                    if mode == "tick" and consecutive_seen >= seen_threshold:
+                        early_out = True
+                        break
+
+                # Persist ledger changes AFTER the network fetch, on short
+                # sessions: mark downloaded/on-disk seen, clear any dead-letter
+                # for recovered media, and record failures (plan #705 #7).
+                if to_mark:
+                    self._mark_seen(source_id, to_mark)
+                if to_clear:
+                    self._clear_failures(source_id, to_clear)
+                if to_fail:
+                    self._record_failures(source_id, to_fail)
+
+                # plan #709: time-throttled live progress to the running event so
+                # the Downloads view ticks ~every 5s, independent of page size.
+                now = time.monotonic()
+                if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL:
+                    last_live = now
+                    self._write_live_progress(event_id, {
+                        "downloaded": downloaded,
+                        "skipped": skipped_count,
+                        "errors": errors,
+                        "quarantined": quarantined,
+                        "posts": posts_processed,
+                    })
+
+                if early_out:
+                    break
+            else:
+                reached_bottom = True
+        except self._error_base as exc:
+            # The platform's client-error base — _failure_result (adapter)
+            # maps it to a typed error.
+            return self._failure_result(exc, _result)
+
+        # plan #708 B4: a Stop already popped the backfill state (incl. cursor +
+        # posts), so don't re-write them — return PARTIAL (reads as "ok/progress",
+        # the lifecycle no-ops since state is gone) instead of a false "complete".
+        if stopped:
+            log.info(
+                "%s ingest STOPPED by operator (%s, source=%s): %d file(s) this chunk",
+                self._platform, mode, source_id, downloaded,
+            )
+            return _result(
+                success=False, return_code=-1,
+                error_type=ErrorType.PARTIAL,
+                error_message=f"Stopped by operator: {downloaded} file(s) this chunk",
+            )
+
+        # Final authoritative posts count for the badge — captures the last page
+        # after the last boundary write and the time-box break (plan #704 #5).
+        if checkpoint:
+            self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
+
+        if errors:
+            log_lines.append(f"{errors} media item(s) failed")
+        if quarantined:
+            log_lines.append(f"{quarantined} media item(s) quarantined (invalid)")
+        if dead_lettered:
+            log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)")
+        summary = (
+            f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
+            f"{skipped_count} skipped, {quarantined} quarantined, "
+            f"{dead_lettered} dead-lettered, {errors} error(s), "
+            f"{posts_processed} post(s), {len(post_records)} post-record(s), "
+            f"{len(relink)} relinked"
+            # Body-capture health (#862): even below the canary's red-alarm
+            # threshold, surfacing the ratio makes a partial extraction regression
+            # visible in the Raw stdout (e.g. "bodies 3/180" reads as off).
+            + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "")
+            + (f", {gated_skipped} gated-skipped" if gated_skipped else "")
+            + (", reached end" if reached_bottom else "")
+            + (", time-boxed" if budget_hit else "")
+        )
+        log_lines.append(summary)
+        # #899 L1: also to the container log (survives event-finalization failure).
+        log.info("%s (source=%s)", summary, source_id)
+
+        if budget_hit:
+            # A chunk that hit its time-box but made forward progress is a
+            # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the
+            # next chunk resumes from the emitted cursor. No progress → TIMEOUT,
+            # which feeds download_service's backfill stall-guard. rc<0 mirrors
+            # subprocess TimeoutExpired so completion detection stays false.
+            made_progress = downloaded > 0 or emitted_cursor != resume_cursor
+            if made_progress:
+                return _result(
+                    success=False, return_code=-1,
+                    error_type=ErrorType.PARTIAL,
+                    error_message=(
+                        f"Backfill chunk: {downloaded} file(s) — continuing"
+                    ),
+                )
+            return _result(
+                success=False, return_code=-1,
+                error_type=ErrorType.TIMEOUT,
+                error_message="Chunk timed out with no progress",
+            )
+
+        # Post-body schema-drift canary (#862): a native walk recorded a
+        # meaningful sample of posts but extracted a body from NONE of them. Since
+        # the body field has no post_type gate, that's the signature of Patreon
+        # renaming/restructuring the body field (as content→content_json_string
+        # already was) — fail RED (API_DRIFT: "fix is the field-set/parser, not
+        # creds") so the breakage screams instead of silently archiving empties.
+        # Only reached on an otherwise-clean walk (timeout/stop/error returned
+        # above), so it never masks a more specific failure.
+        if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0:
+            msg = (
+                f"Post-body canary: extracted a body from 0 of {posts_recorded} "
+                "posts — Patreon's body field shape likely changed; the ingester "
+                "needs a field-set/parser update."
+            )
+            log_lines.append(msg)
+            log.error("%s (artist=%s)", msg, artist_slug)
+            return _result(
+                success=False, return_code=-1,
+                error_type=ErrorType.API_DRIFT, error_message=msg,
+            )
+
+        # Normal success: reached the bottom, or a tick that early-outed. rc 0 +
+        # error_type None is REQUIRED for a backfill/recovery walk that reached
+        # the bottom to be marked COMPLETE by
+        # download_service._apply_backfill_lifecycle — so we return None even
+        # when downloaded == 0 (a re-confirming walk that found nothing new still
+        # completed). success=True maps to status "ok" regardless. A tick that
+        # early-outed also returns here; ticks never set backfill state so the
+        # lifecycle is a no-op for them.
+        return _result(
+            success=True, return_code=0,
+            error_type=None, error_message=None,
+        )
+
+    # -- preview (dry-run) -------------------------------------------------
+
+    def preview(
+        self,
+        source_id: int,
+        campaign_id: str,
+        *,
+        page_limit: int = 3,
+        sample_size: int = 10,
+    ) -> dict:
+        """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
+        NOT already in the seen/dead ledgers, WITHOUT downloading anything.
+
+        Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
+        an operator gauge "is this source worth a backfill?" cheaply. Returns:
+            {total_new, posts_scanned, pages_scanned, has_more,
+             sample: [{title, date, new}, ...]}  # sample = posts with new media
+        A client-level failure (auth/drift) propagates to the caller.
+        """
+        total_new = 0
+        posts_scanned = 0
+        pages_scanned = 0
+        has_more = False
+        sample: list[dict] = []
+        unset = object()
+        last_page: object = unset
+        # #874: same gated-post gate as run() — the preview must not count
+        # blurred locked-preview media as "new", or it would overstate a gated
+        # source's backlog (preview/apply parity, rule 93).
+        post_is_gated = getattr(self.client, "post_is_gated", None)
+        for post, included, page_cursor in self.client.iter_posts(
+            campaign_id, cursor=None
+        ):
+            if page_cursor != last_page:
+                last_page = page_cursor
+                pages_scanned += 1
+                if pages_scanned > page_limit:
+                    has_more = True
+                    pages_scanned = page_limit
+                    break
+            posts_scanned += 1
+            if post_is_gated and post_is_gated(post):
+                continue
+            media = self.client.extract_media(post, included)
+            if not media:
+                continue
+            keys = [self._ledger_key(m) for m in media]
+            skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
+            new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
+            total_new += new_count
+            if new_count > 0 and len(sample) < sample_size:
+                meta = self.client.post_meta(post)
+                sample.append(
+                    {
+                        "title": meta.get("title") or "(untitled)",
+                        "date": meta.get("date"),
+                        "new": new_count,
+                    }
+                )
+        return {
+            "total_new": total_new,
+            "posts_scanned": posts_scanned,
+            "pages_scanned": pages_scanned,
+            "has_more": has_more,
+            "sample": sample,
+        }
+
+    # -- failure mapping (adapter overrides) -------------------------------
+
+    def _failure_result(self, exc: Exception, _result) -> DownloadResult:
+        """Map a platform client-error to a loud, typed failed DownloadResult —
+        NEVER a silent zero-download "success". The mapping is shared across
+        platforms via the NativeAuthError/NativeDriftError taxonomy (the platform
+        client raises subclasses), so a new platform gets it for free:
+          - NativeAuthError  → AUTH_ERROR   (rotate the credential)
+          - NativeDriftError → API_DRIFT    (the ingester/scraper needs updating)
+          - HTTP 429 / 404   → RATE_LIMITED / NOT_FOUND
+          - other HTTP status→ HTTP_ERROR;  transport failure → NETWORK_ERROR
+        Auth/Drift are matched first (they also carry a status_code in some paths).
+        """
+        message = str(exc)
+        if isinstance(exc, NativeAuthError):
+            error_type = ErrorType.AUTH_ERROR
+        elif isinstance(exc, NativeDriftError):
+            error_type = ErrorType.API_DRIFT
+            message = f"{self._drift_label} changed — ingester needs update: {message}"
+        else:
+            status = getattr(exc, "status_code", None)
+            if status == 429:
+                error_type = ErrorType.RATE_LIMITED
+            elif status == 404:
+                error_type = ErrorType.NOT_FOUND
+            elif status is not None:
+                error_type = ErrorType.HTTP_ERROR
+            else:
+                error_type = ErrorType.NETWORK_ERROR
+        log.warning("%s ingest failed (%s): %s", self._platform, error_type.value, message)
+        result = _result(
+            success=False, return_code=1,
+            error_type=error_type, error_message=message,
+        )
+        # plan #708 B1: carry the server's Retry-After up to the cooldown.
+        if error_type == ErrorType.RATE_LIMITED:
+            result.retry_after_seconds = getattr(exc, "retry_after", None)
+        return result
+
+    # -- seen-ledger (short-lived sessions) --------------------------------
+
+    def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]:
+        """Which of `keys` are already in the seen-ledger for this source.
+
+        One short SELECT on its own session — opened and closed without any
+        network in between (the GETs happen after, in download_post).
+        """
+        if not keys:
+            return set()
+        with self.session_factory() as session:
+            rows = session.execute(
+                select(self._seen_model.filehash).where(
+                    self._seen_model.source_id == source_id,
+                    self._seen_model.filehash.in_(keys),
+                )
+            ).scalars().all()
+        return set(rows)
+
+    def _checkpoint_cursor(self, source_id: int, cursor: str) -> None:
+        """Persist the in-progress backfill cursor mid-walk (plan #705 #6).
+
+        ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just
+        `_backfill_cursor`, cast back — so it never clobbers operator config or
+        the other backfill keys (no read-modify-write race). The in-flight guard
+        means only this source's one download runs at a time; a concurrent
+        operator stop is benign (a stray cursor with no `_backfill_state` is
+        ignored by tick mode and cleared on the next start).
+        """
+        with self.session_factory() as session:
+            session.execute(
+                text(
+                    "UPDATE source SET config_overrides = jsonb_set("
+                    "  coalesce(config_overrides::jsonb, '{}'::jsonb),"
+                    "  '{_backfill_cursor}', to_jsonb(cast(:cur AS text))"
+                    ")::json WHERE id = :sid"
+                ),
+                {"cur": cursor, "sid": source_id},
+            )
+            session.commit()
+
+    def _write_live_progress(self, event_id: int, counts: dict) -> None:
+        """Throttled mid-walk write of live counts to the RUNNING download_event
+        (plan #709) so the Downloads view shows progress before the chunk
+        finishes. A short session (never held across the walk); the `status =
+        'running'` guard avoids clobbering an event phase 3 already finalized.
+        `metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest
+        for phase 3 to overwrite with the final run_stats."""
+        with self.session_factory() as session:
+            session.execute(
+                text(
+                    "UPDATE download_event SET metadata = jsonb_set("
+                    "  coalesce(metadata, '{}'::jsonb), '{live}',"
+                    "  cast(:live AS jsonb)) "
+                    "WHERE id = :eid AND status = 'running'"
+                ),
+                {"live": json.dumps(counts), "eid": event_id},
+            )
+            session.commit()
+
+    def _still_running(self, source_id: int) -> bool:
+        """True while the source is armed for a deep walk (plan #708 B4).
+
+        An operator Stop (`source_service.stop_backfill`) pops `_backfill_state`,
+        so a False here means "cancel this chunk now". One short SELECT on its own
+        session — never held across the walk
+        ([[db-connection-held-across-subprocess]])."""
+        with self.session_factory() as session:
+            state = session.execute(
+                text(
+                    "SELECT config_overrides::jsonb ->> '_backfill_state' "
+                    "FROM source WHERE id = :sid"
+                ),
+                {"sid": source_id},
+            ).scalar_one_or_none()
+        return state == "running"
+
+    def _checkpoint_posts(self, source_id: int, posts: int) -> None:
+        """Persist the live backfill posts-processed count mid-walk (plan #704 #5).
+
+        Same atomic single-key jsonb_set dance as _checkpoint_cursor, on the
+        `_backfill_posts` key (cast to a JSON number) — so the progress badge
+        climbs DURING a chunk without clobbering operator config or the cursor.
+        The ingester OWNS this key now; download_service no longer accumulates it
+        post-chunk (which lagged a whole chunk and over-counted the resume page).
+        """
+        with self.session_factory() as session:
+            session.execute(
+                text(
+                    "UPDATE source SET config_overrides = jsonb_set("
+                    "  coalesce(config_overrides::jsonb, '{}'::jsonb),"
+                    "  '{_backfill_posts}', to_jsonb(cast(:posts AS int))"
+                    ")::json WHERE id = :sid"
+                ),
+                {"posts": posts, "sid": source_id},
+            )
+            session.commit()
+
+    def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
+        """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
+
+        ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a
+        re-sighting — or a concurrent walk — is a harmless no-op
+        ([[scalar_one_or_none-duplicates]]: never check-then-insert without the
+        DB constraint backing it). De-dup the batch locally first so a single
+        page can't present the same key twice to one INSERT.
+        """
+        seen_local: set[str] = set()
+        values = []
+        for key, post_id in items:
+            if key in seen_local:
+                continue
+            seen_local.add(key)
+            values.append(
+                {"source_id": source_id, "filehash": key, "post_id": post_id}
+            )
+        if not values:
+            return
+        with self.session_factory() as session:
+            stmt = pg_insert(self._seen_model).values(values)
+            stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint)
+            session.execute(stmt)
+            session.commit()
+
+    # -- dead-letter ledger (plan #705 #7) ---------------------------------
+
+    def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]:
+        """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead).
+        One short SELECT; recovery never calls this (it re-attempts dead media)."""
+        if not keys:
+            return set()
+        with self.session_factory() as session:
+            rows = session.execute(
+                select(self._failed_model.filehash).where(
+                    self._failed_model.source_id == source_id,
+                    self._failed_model.filehash.in_(keys),
+                    self._failed_model.attempts >= DEAD_LETTER_THRESHOLD,
+                )
+            ).scalars().all()
+        return set(rows)
+
+    def _record_failures(
+        self, source_id: int, items: list[tuple[str, str, str]]
+    ) -> None:
+        """Upsert-increment the dead-letter ledger for failed media. On conflict
+        bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the
+        upsert — no check-then-insert). De-dup the batch (one row/key, last error
+        wins)."""
+        by_key: dict[str, str] = {}
+        for key, _post_id, err in items:
+            by_key[key] = (err or "")[:_ERROR_MAX]
+        if not by_key:
+            return
+        values = [
+            {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e}
+            for k, e in by_key.items()
+        ]
+        with self.session_factory() as session:
+            stmt = pg_insert(self._failed_model).values(values)
+            stmt = stmt.on_conflict_do_update(
+                constraint=self._failed_constraint,
+                set_={
+                    "attempts": self._failed_model.attempts + 1,
+                    "last_error": stmt.excluded.last_error,
+                    "last_failed_at": func.now(),
+                },
+            )
+            session.execute(stmt)
+            session.commit()
+
+    def _clear_failures(self, source_id: int, keys: list[str]) -> None:
+        """Drop dead-letter rows for media that just downloaded cleanly — they
+        recovered. A no-op DELETE for keys that were never failing."""
+        unique = list(dict.fromkeys(keys))
+        if not unique:
+            return
+        with self.session_factory() as session:
+            session.execute(
+                delete(self._failed_model).where(
+                    self._failed_model.source_id == source_id,
+                    self._failed_model.filehash.in_(unique),
+                )
+            )
+            session.commit()

backend/app/services/link_extract.py

@@ -0,0 +1,105 @@
+"""Extract off-platform file-host links from a post body.
+
+Pure (no I/O) so it's unit-testable and reusable by EVERY in-house downloader's
+import path — every platform stores its body in `Post.description`, so running
+this there covers them all. Finds links to the supported external file hosts in
+a post's HTML body (both `<a href="...">` anchors and bare URLs in text),
+unwraps a Patreon outbound-redirect wrapper, and preserves the FULL url
+including the `#fragment` (mega.nz puts the decryption key there) and the query
+string — without those a mega download is impossible.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from html import unescape
+from urllib.parse import parse_qs, urlsplit
+
+# Supported file-host enum values (kept in sync with the external_link CHECK).
+SUPPORTED_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain")
+
+# Bare-domain suffix → canonical host label. Matched against the URL netloc
+# (exact or as a dotted suffix, so www./dl. subdomains resolve too).
+_HOST_MAP = {
+    "mega.nz": "mega",
+    "mega.co.nz": "mega",
+    "drive.google.com": "gdrive",
+    "mediafire.com": "mediafire",
+    "dropbox.com": "dropbox",
+    "pixeldrain.com": "pixeldrain",
+}
+
+# `<a href="...">label</a>` — DOTALL so a label spanning tags/newlines is caught.
+_HREF_RE = re.compile(
+    r"""<a\b[^>]*?\bhref=["']([^"']+)["'][^>]*>(.*?)</a>""",
+    re.IGNORECASE | re.DOTALL,
+)
+_TAG_RE = re.compile(r"<[^>]+>")
+# Bare http(s) URL in text. Stops at whitespace, quotes, angle brackets, and
+# closing brackets — but KEEPS `#`, `&`, `?`, `=` so fragments/queries survive.
+_URL_RE = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class ExtractedLink:
+    host: str            # one of SUPPORTED_HOSTS
+    url: str             # full url incl. query + #fragment
+    label: str | None    # visible anchor text, when present
+
+
+def _netloc(url: str) -> str:
+    # Lowercase host without credentials or port.
+    return urlsplit(url).netloc.lower().split("@")[-1].split(":")[0]
+
+
+def host_for(url: str) -> str | None:
+    """Canonical host label for a url, or None if it's not a supported host."""
+    netloc = _netloc(url)
+    for suffix, host in _HOST_MAP.items():
+        if netloc == suffix or netloc.endswith("." + suffix):
+            return host
+    return None
+
+
+def _unwrap(url: str) -> str:
+    """Unwrap a Patreon outbound-redirect wrapper to its inner target, so a
+    wrapped mega/gdrive link resolves to the real host. Patreon has used both
+    `www.patreon.com/...?url=<encoded>` and the `l.patreon.com` shim; check the
+    common target-param names. Non-Patreon urls pass through untouched."""
+    netloc = _netloc(url)
+    if not (netloc == "patreon.com" or netloc.endswith(".patreon.com")):
+        return url
+    qs = parse_qs(urlsplit(url).query)
+    for key in ("url", "u", "ext_url", "redirect", "target"):
+        vals = qs.get(key)
+        if vals and vals[0]:
+            return unescape(vals[0]).strip()
+    return url
+
+
+def extract_external_links(html: str | None) -> list[ExtractedLink]:
+    """All supported-host links in `html`, de-duplicated by url (first wins, so
+    an anchor's label is kept over a later bare sighting of the same url)."""
+    if not html:
+        return []
+    found: dict[str, ExtractedLink] = {}
+
+    # 1) Anchors first — they carry a human label ("Mega - Streamable").
+    for raw_href, inner in _HREF_RE.findall(html):
+        url = _unwrap(unescape(raw_href).strip())
+        host = host_for(url)
+        if host is None:
+            continue
+        label = unescape(_TAG_RE.sub("", inner)).strip() or None
+        found.setdefault(url, ExtractedLink(host=host, url=url, label=label))
+
+    # 2) Bare URLs pasted in text (no label). Trailing prose punctuation is
+    #    trimmed; the href values already captured above de-dup away here.
+    for raw in _URL_RE.findall(html):
+        url = _unwrap(unescape(raw).strip().rstrip(".,;"))
+        host = host_for(url)
+        if host is None:
+            continue
+        found.setdefault(url, ExtractedLink(host=host, url=url, label=None))
+
+    return list(found.values())

backend/app/services/ml/aliases.py

@@ -1,7 +1,7 @@
 """Alias resolution + CRUD.
 
 A tag_alias maps (model_name, model_category) -> canonical Tag. Resolution
-happens at suggestion-read time so raw tagger_predictions stay unmolested.
+happens at suggestion-read time so the raw image_prediction rows stay unmolested.
 """
 
 from collections.abc import Sequence
@@ -81,6 +81,31 @@ class AliasService:
             .where(TagAlias.alias_category == alias_category)
         )
 
+    async def list_for_tag(self, canonical_tag_id: int) -> Sequence[AliasRow]:
+        """Aliases that resolve TO this tag — drives the tag-side 'Aliases'
+        view (see/remove the model keys that fold into a tag)."""
+        stmt = (
+            select(
+                TagAlias.alias_string,
+                TagAlias.alias_category,
+                TagAlias.canonical_tag_id,
+                Tag.name,
+            )
+            .join(Tag, Tag.id == TagAlias.canonical_tag_id)
+            .where(TagAlias.canonical_tag_id == canonical_tag_id)
+            .order_by(TagAlias.alias_string.asc())
+        )
+        rows = (await self.session.execute(stmt)).all()
+        return [
+            AliasRow(
+                alias_string=r[0],
+                alias_category=r[1],
+                canonical_tag_id=r[2],
+                canonical_tag_name=r[3],
+            )
+            for r in rows
+        ]
+
     async def list_all(self) -> Sequence[AliasRow]:
         stmt = (
             select(

backend/app/services/ml/allowlist.py

@@ -5,11 +5,18 @@ image_tag AND to tag_allowlist; per-image removal/dismiss writes a rejection.
 from collections.abc import Sequence
 from dataclasses import dataclass
 
-from sqlalchemy import delete, select
+from sqlalchemy import and_, delete, distinct, func, or_, select
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ...models import Tag, TagAllowlist, TagSuggestionRejection
+from ...models import (
+    ImagePrediction,
+    MLSettings,
+    Tag,
+    TagAlias,
+    TagAllowlist,
+    TagSuggestionRejection,
+)
 from ...models.tag import image_tag
 from .aliases import AliasService
 
@@ -20,6 +27,8 @@ class AllowlistRow:
     tag_name: str
     tag_kind: str
     min_confidence: float
+    applied_count: int      # image_tag rows currently carrying this tag
+    coverage_count: int     # images a sweep WOULD cover at min_confidence
 
 
 class AllowlistService:
@@ -91,18 +100,69 @@ class AllowlistService:
         )
         await self.dismiss(image_id, tag_id)
 
+    async def _store_floor(self) -> float:
+        return (
+            await self.session.execute(
+                select(MLSettings.tagger_store_floor).where(MLSettings.id == 1)
+            )
+        ).scalar_one()
+
     async def update_threshold(
         self, tag_id: int, min_confidence: float
     ) -> None:
         row = await self.session.get(TagAllowlist, tag_id)
         if row is not None:
-            row.min_confidence = min_confidence
+            # An allowlist tag can't auto-apply more permissively than the
+            # ingest store floor — predictions below tagger_store_floor aren't
+            # stored, so a lower min_confidence would behave identically to the
+            # floor. Clamp so the stored threshold matches actual behavior
+            # (#764).
+            floor = await self._store_floor()
+            row.min_confidence = max(min_confidence, floor)
 
     async def remove(self, tag_id: int) -> None:
         await self.session.execute(
             delete(TagAllowlist).where(TagAllowlist.tag_id == tag_id)
         )
 
+    async def _coverage_match(self, tag: Tag):
+        """The predicate over image_prediction rows that resolve to `tag`,
+        mirroring tasks.ml._confidence_for_tag's resolution: a prediction whose
+        raw_name equals the tag name (any category), OR an alias maps
+        (raw_name, category) -> this tag. Returns a SQLAlchemy boolean clause.
+        """
+        alias_rows = (
+            await self.session.execute(
+                select(TagAlias.alias_string, TagAlias.alias_category).where(
+                    TagAlias.canonical_tag_id == tag.id
+                )
+            )
+        ).all()
+        name_clause = ImagePrediction.raw_name == tag.name
+        alias_clauses = [
+            and_(
+                ImagePrediction.raw_name == a,
+                ImagePrediction.category == c,
+            )
+            for a, c in alias_rows
+        ]
+        return or_(name_clause, *alias_clauses) if alias_clauses else name_clause
+
+    async def coverage(self, tag_id: int, threshold: float) -> int:
+        """How many distinct images a sweep WOULD cover for this tag at
+        `threshold`: images with a resolving prediction scoring >= threshold.
+        The gross candidate pool (NOT minus already-applied/rejected) — it's
+        the tuning signal for "lower the threshold and ~N more images qualify".
+        """
+        tag = await self.session.get(Tag, tag_id)
+        if tag is None:
+            return 0
+        match = await self._coverage_match(tag)
+        stmt = select(
+            func.count(distinct(ImagePrediction.image_record_id))
+        ).where(ImagePrediction.score >= threshold, match)
+        return (await self.session.execute(stmt)).scalar_one()
+
     async def list_all(self) -> Sequence[AllowlistRow]:
         stmt = (
             select(
@@ -115,12 +175,33 @@ class AllowlistService:
             .order_by(Tag.name.asc())
         )
         rows = (await self.session.execute(stmt)).all()
-        return [
-            AllowlistRow(
-                tag_id=r[0],
-                tag_name=r[1],
-                tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]),
-                min_confidence=r[3],
+        tag_ids = [r[0] for r in rows]
+
+        # Applied counts in ONE grouped query (vs N per-row counts).
+        applied: dict[int, int] = {}
+        if tag_ids:
+            applied = dict(
+                (
+                    await self.session.execute(
+                        select(image_tag.c.tag_id, func.count())
+                        .where(image_tag.c.tag_id.in_(tag_ids))
+                        .group_by(image_tag.c.tag_id)
+                    )
+                ).all()
             )
-            for r in rows
-        ]
+
+        result = []
+        for r in rows:
+            # Coverage is per-tag (alias set differs); allowlist is small.
+            cov = await self.coverage(r[0], r[3])
+            result.append(
+                AllowlistRow(
+                    tag_id=r[0],
+                    tag_name=r[1],
+                    tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]),
+                    min_confidence=r[3],
+                    applied_count=applied.get(r[0], 0),
+                    coverage_count=cov,
+                )
+            )
+        return result

backend/app/services/ml/embedder.py

@@ -1,8 +1,8 @@
 """SigLIP SO400M image-embedding wrapper (PyTorch CPU).
 
-Direct port of ImageRepo's siglip.py. torch/transformers are imported
-lazily inside load() so this module can be imported in the web container
-(which never runs inference) without paying the torch import cost.
+torch/transformers are imported lazily inside load() so this module can be
+imported in the web container (which never runs inference) without paying the
+torch import cost.
 """
 
 import os
@@ -13,6 +13,11 @@ from PIL import Image, ImageFile
 
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
+# Cap torch's intra-op threads so each ml-worker replica is a bounded core
+# consumer on a shared node (torch otherwise uses all cores). Keep
+# N_replicas × this within the cores allotted to ML to avoid oversubscription.
+_INTRA_OP_THREADS = 4
+
 MODEL_NAME = os.environ.get(
     "SIGLIP_MODEL_NAME", "google/siglip-so400m-patch14-384"
 )
@@ -37,6 +42,9 @@ class Embedder:
         from transformers import AutoModel, SiglipImageProcessor
 
         self._torch = torch
+        # Bound torch's CPU thread pool (see _INTRA_OP_THREADS) so each replica
+        # stays a predictable core consumer on a shared node.
+        torch.set_num_threads(_INTRA_OP_THREADS)
         # FC's embedder only does IMAGE inference — never text. AutoProcessor
         # loads the full processor including SiglipTokenizer, which requires
         # the sentencepiece library at import time even if we never call it.

backend/app/services/ml/suggestions.py

@@ -8,6 +8,7 @@ from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ...models import (
+    ImagePrediction,
     ImageRecord,
     MLSettings,
     Tag,
@@ -30,6 +31,14 @@ class Suggestion:
     score: float
     source: str  # 'tagger' | 'centroid' | 'both'
     creates_new_tag: bool
+    # raw_name = the booru model vocab key behind this suggestion. It's the key
+    # an alias MUST be stored under (resolution looks up the raw key), so the
+    # modal needs it to author an alias correctly. None for centroid-only hits
+    # (no underlying prediction → nothing to alias).
+    raw_name: str | None = None
+    # via_alias = this suggestion was surfaced because an operator alias remapped
+    # the raw prediction to this canonical tag. Lets the UI mark it + offer undo.
+    via_alias: bool = False
 
 
 @dataclass
@@ -48,22 +57,58 @@ class SuggestionService:
             await self.session.execute(select(MLSettings).where(MLSettings.id == 1))
         ).scalar_one()
 
-    def _threshold_for(self, s: MLSettings, category: str) -> float:
+    async def _load_predictions(self, image_id: int) -> dict:
+        """Predictions for one image from the normalized image_prediction
+        table (#768), in the {raw_name: {category, confidence}} shape the rest
+        of this service consumed from the old JSON column — so all downstream
+        threshold/alias/merge logic is unchanged."""
+        rows = (
+            await self.session.execute(
+                select(
+                    ImagePrediction.raw_name,
+                    ImagePrediction.category,
+                    ImagePrediction.score,
+                ).where(ImagePrediction.image_record_id == image_id)
+            )
+        ).all()
+        return {
+            r.raw_name: {"category": r.category, "confidence": r.score}
+            for r in rows
+        }
+
+    def _threshold_for(
+        self, s: MLSettings, category: str, override: float | None = None,
+    ) -> float:
         # 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired;
         # both fall through to the 1.01 "never surfaces" default like any
         # unsurfaced category.
+        # override (the typed-dropdown "show everything the model saw" mode)
+        # applies to the surfaced categories only — unsurfaced ones are already
+        # skipped before the threshold check, so they can't leak in.
+        if override is not None:
+            return override
         return {
             "character": s.suggestion_threshold_character,
             "general": s.suggestion_threshold_general,
         }.get(category, 1.01)
 
-    async def for_image(self, image_id: int) -> SuggestionList:
+    async def for_image(
+        self, image_id: int, *, threshold_override: float | None = None,
+    ) -> SuggestionList:
+        """Ranked suggestions for one image.
+
+        threshold_override surfaces EVERY stored tagger prediction (down to the
+        ingest STORE_FLOOR) regardless of the configured per-category suggestion
+        thresholds — backs the tag-input dropdown's "search all of the model's
+        predictions, including low-confidence ones, in the canonical formatting"
+        mode (operator-asked 2026-06-09). The Suggestions panel still calls with
+        no override so it stays the curated above-threshold list."""
         img = await self.session.get(ImageRecord, image_id)
         if img is None:
             return SuggestionList()
 
         settings = await self._settings()
-        predictions: dict = img.tagger_predictions or {}
+        predictions: dict = await self._load_predictions(image_id)
 
         applied = set(
             (
@@ -96,7 +141,7 @@ class SuggestionService:
             if category not in SURFACED_CATEGORIES:
                 continue
             conf = float(p.get("confidence", 0.0))
-            if conf < self._threshold_for(settings, category):
+            if conf < self._threshold_for(settings, category, threshold_override):
                 continue
             display = normalize_tag_name(name)
             if display is None:
@@ -124,6 +169,11 @@ class SuggestionService:
                     if existing.source != sug.source
                     else existing.source,
                     creates_new_tag=existing.creates_new_tag,
+                    # Keep the alias identity from `existing`: the tagger pass
+                    # (which carries raw_name / via_alias) runs before centroid
+                    # augmentation, so it's always the first writer for a key.
+                    raw_name=existing.raw_name,
+                    via_alias=existing.via_alias,
                 )
 
         for raw, display, category, conf in candidates:
@@ -140,6 +190,8 @@ class SuggestionService:
                         score=conf,
                         source="tagger",
                         creates_new_tag=False,
+                        raw_name=raw,
+                        via_alias=True,
                     ),
                 )
             else:
@@ -171,6 +223,8 @@ class SuggestionService:
                             score=conf,
                             source="tagger",
                             creates_new_tag=False,
+                            raw_name=raw,
+                            via_alias=False,
                         ),
                     )
                 else:
@@ -183,6 +237,8 @@ class SuggestionService:
                             score=conf,
                             source="tagger",
                             creates_new_tag=True,
+                            raw_name=raw,
+                            via_alias=False,
                         ),
                     )
 

backend/app/services/ml/tagger.py

@@ -1,8 +1,8 @@
-"""Camie-tagger-v2 ONNX wrapper.
+"""Camie-tagger-v2 ONNX wrapper (CPU).
 
-CPU-only, single-image at a time. Loaded lazily inside the ml-worker
-process; NOT thread-safe — the ml queue worker must run --concurrency=1
-(set by the FC-1 entrypoint).
+Single-image at a time. Loaded lazily inside the ml-worker process; NOT
+thread-safe — the ml queue worker runs --concurrency=1 per process (scale ML by
+running multiple worker replicas, not threads).
 
 v2 layout reference: HuggingFace Camais03/camie-tagger-v2 root has
 camie-tagger-v2.onnx (789 MB) + camie-tagger-v2-metadata.json (7.77 MB)
@@ -19,6 +19,11 @@ from pathlib import Path
 import numpy as np
 from PIL import Image, ImageFile
 
+# Cap inference threads (see Tagger.load) so each ml-worker replica is a bounded
+# core consumer on a shared node — keep N_replicas × this within the cores
+# allotted to ML so replicas don't oversubscribe the box / starve the DB.
+_INTRA_OP_THREADS = 4
+
 # onnxruntime lives in requirements-ml.txt only — it is NOT installed in the
 # lean web image or in CI. Imported lazily inside Tagger.load() so this module
 # imports fine without it (the suggestion service imports SURFACED_CATEGORIES
@@ -33,8 +38,13 @@ _MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
 _MODEL_FILE = f"{MODEL_NAME}.onnx"
 _METADATA_FILE = f"{MODEL_NAME}-metadata.json"
 
-# Below this confidence, predictions aren't stored (keeps the JSON compact).
-STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
+# Ingest floor below which predictions aren't stored (keeps the JSON compact).
+# DEFAULT/fallback only — the live value is DB-backed
+# (ml_settings.tagger_store_floor) and passed into infer() per call by the ml
+# task. 0.70: the suggestion path already filters there and the centroid path
+# covers lower-confidence preferred tags, so the sub-0.70 tail is redundant
+# (it had bloated image_record's TOAST to ~100 GB; plan-task #764).
+DEFAULT_STORE_FLOOR = 0.70
 
 # The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
 # still stored but the suggestion service filters them out.
@@ -112,8 +122,15 @@ class Tagger:
         # without onnxruntime (CI / lean web image).
         import onnxruntime as ort
 
+        # Cap the intra-op thread pool. ONNX Runtime otherwise sizes it to ALL
+        # host cores, so on a shared node each ml-worker replica would grab every
+        # core and oversubscribe (and starve the co-located DB/web). Bounding it
+        # makes each replica a predictable core consumer — run N replicas where
+        # N × _INTRA_OP_THREADS stays within the cores you allot to ML.
+        opts = ort.SessionOptions()
+        opts.intra_op_num_threads = _INTRA_OP_THREADS
         session = ort.InferenceSession(
-            str(model_path), providers=["CPUExecutionProvider"]
+            str(model_path), sess_options=opts, providers=["CPUExecutionProvider"],
         )
         self._input_name = session.get_inputs()[0].name
         # Assign sentinels last so a partial load isn't observable.
@@ -145,10 +162,13 @@ class Tagger:
         arr = arr.transpose(2, 0, 1)                        # HWC -> CHW
         return arr[np.newaxis, :, :, :]                     # NCHW
 
-    def infer(self, image_path: Path) -> dict[str, TagPrediction]:
+    def infer(
+        self, image_path: Path, *, store_floor: float = DEFAULT_STORE_FLOOR,
+    ) -> dict[str, TagPrediction]:
         """Run Camie v2 on one image. Returns {name: TagPrediction} with
-        confidence >= STORE_FLOOR (across all categories — the suggestion
-        service does category filtering later).
+        confidence >= store_floor (across all categories — the suggestion
+        service does category filtering later). store_floor is the DB-backed
+        ml_settings.tagger_store_floor, passed in by the ml task.
 
         v2 emits multiple outputs; we use the refined predictions
         (output[1] per onnx_inference.py). Sigmoid is applied to raw
@@ -167,7 +187,7 @@ class Tagger:
         cats = self._tag_categories
         for idx, score in enumerate(probs):
             conf = float(score)
-            if conf < STORE_FLOOR:
+            if conf < store_floor:
                 continue
             if idx >= len(names):
                 # Output longer than metadata declared — shouldn't happen but

backend/app/services/native_ingest_common.py

@@ -0,0 +1,345 @@
+"""Shared primitives for the native-ingest platform adapters (Patreon,
+SubscribeStar, …) — the single home for logic the per-platform client/downloader
+modules would otherwise each copy.
+
+DRY pass 2026-06-17 (#899): these used to live in `patreon_*` with the
+SubscribeStar modules importing patreon privates (wrong owner + sibling-coupling).
+They're platform-agnostic, so they live here and both adapters import them. The
+per-platform modules keep only what genuinely differs (feed parsing, the media
+shape, Patreon's Mux/yt-dlp video branch + detail-fetch enrichment).
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import http.cookiejar
+import json
+import logging
+import os
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import requests
+
+from ..utils.paths import filehash_from_url, safe_ext
+from .file_validator import is_validatable, quarantine_file, validate_file
+
+log = logging.getLogger(__name__)
+
+_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+)
+
+# 429 backoff (plan #703): ride out a transient API rate-limit instead of failing
+# the whole walk. Honor the server's Retry-After; else exponential, capped.
+_MAX_429_RETRIES = 3
+_BACKOFF_BASE_SECONDS = 2.0
+_BACKOFF_CAP_SECONDS = 30.0
+
+# Media-download tuning (shared by every platform downloader).
+_TIMEOUT_SECONDS = 120.0
+_CHUNK = 1 << 16
+_MAX_MEDIA_RETRIES = 3
+_TRANSIENT_TRANSPORT_EXC = (
+    requests.ConnectionError,
+    requests.Timeout,
+    requests.exceptions.ChunkedEncodingError,
+)
+
+_TITLE_MAX = 40
+# Windows/gallery-dl path-restrict forbidden set + path separators.
+_FORBIDDEN = set('<>:"/\\|?*')
+
+
+# -- shared exception taxonomy --------------------------------------------
+# Every native client raises one of these (platform subclasses keep an
+# isinstance-distinct platform name AND the semantic Auth/Drift class), so the
+# base Ingester._failure_result can map them platform-agnostically.
+
+class NativeIngestError(Exception):
+    """Base for a native-ingest client failure. `status_code` carries the HTTP
+    status when the failure was an HTTP response (None for transport/parse);
+    `retry_after` carries the server's 429 Retry-After hint so the cooldown can
+    match it (plan #708 B1)."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        retry_after: float | None = None,
+    ):
+        super().__init__(message)
+        self.status_code = status_code
+        self.retry_after = retry_after
+
+
+class NativeAuthError(NativeIngestError):
+    """Authentication/authorization failure — expired/missing credential or an
+    insufficient tier. The fix is rotating the credential, NOT updating the
+    ingester. Maps to error_type 'auth_error'."""
+
+
+class NativeDriftError(NativeIngestError):
+    """A response did not match the shape the ingester depends on (JSON:API field
+    set, or scraped HTML structure). Fail loud so the import step flags 'the
+    platform changed' instead of silently importing nothing. Maps to API_DRIFT."""
+
+
+# -- HTTP session ----------------------------------------------------------
+
+def make_session(
+    cookies_path: str | Path | None,
+    *,
+    accept: str = "*/*",
+    extra_headers: dict | None = None,
+) -> requests.Session:
+    """Build a requests.Session loaded with the Netscape cookies.txt
+    CredentialService materializes. `accept` sets the Accept header (the JSON:API
+    vs HTML feed differ); `extra_headers` adds platform headers (e.g.
+    X-Requested-With). Missing/unparseable cookies log a warning, never fail."""
+    session = requests.Session()
+    headers = {"User-Agent": _USER_AGENT, "Accept": accept}
+    if extra_headers:
+        headers.update(extra_headers)
+    session.headers.update(headers)
+    if cookies_path and os.path.isfile(str(cookies_path)):
+        try:
+            jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
+            jar.load(ignore_discard=True, ignore_expires=True)
+            session.cookies = jar  # type: ignore[assignment]
+        except (OSError, http.cookiejar.LoadError) as exc:
+            log.warning("Could not load cookies from %s: %s", cookies_path, exc)
+    return session
+
+
+def retry_after_seconds(
+    resp: requests.Response,
+    attempt: int,
+    *,
+    base: float = _BACKOFF_BASE_SECONDS,
+    cap: float = _BACKOFF_CAP_SECONDS,
+) -> float:
+    """Backoff for a 429: the numeric Retry-After header if present, else
+    exponential base·2^(attempt-1), both capped."""
+    header = resp.headers.get("Retry-After")
+    if header:
+        try:
+            return min(float(header), cap)
+        except (TypeError, ValueError):
+            pass
+    return min(base * (2 ** max(0, attempt - 1)), cap)
+
+
+# -- filename / path helpers -----------------------------------------------
+
+def sanitize_segment(name: str) -> str:
+    """Make `name` safe for one filesystem path segment: replace separators, the
+    Windows-forbidden set, and control chars with `_`; strip trailing dots/spaces
+    (gallery-dl path-restrict). Never empty (falls back to `_`)."""
+    out = ["_" if (ch in _FORBIDDEN or ord(ch) < 32) else ch for ch in name]
+    cleaned = "".join(out).rstrip(". ")
+    return cleaned or "_"
+
+
+def basename_from_url(url: str) -> str:
+    """Derive a sane filename from a URL when the media has no name: path basename
+    with a junk-extension guard (safe_ext), bounded stem; falls back to the URL's
+    content hash, then "file"."""
+    path = urlsplit(url).path
+    base = os.path.basename(path)
+    if base:
+        ext = safe_ext(base)
+        stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
+        stem = stem[:120] or "file"
+        return f"{stem}{ext}"
+    return filehash_from_url(url) or "file"
+
+
+def post_dir_name(post: dict) -> str:
+    """`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's layout (date prefix
+    omitted when published_at is missing/unparseable; title is empty for platforms
+    with no title field). Accepts both ISO and trailing-`Z` published_at."""
+    post_id = str(post.get("id") or "")
+    attrs = post.get("attributes") or {}
+    title = attrs.get("title")
+    title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
+    published = attrs.get("published_at")
+    date_prefix = None
+    if isinstance(published, str) and published:
+        s = published.strip()
+        if s.endswith("Z"):
+            s = s[:-1] + "+00:00"
+        try:
+            date_prefix = f"{datetime.fromisoformat(s):%Y-%m-%d}"
+        except ValueError:
+            date_prefix = None
+    raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
+    return sanitize_segment(raw)
+
+
+# -- per-item download outcomes (shared dataclasses) -----------------------
+
+@dataclass
+class MediaOutcome:
+    """Per-media result of a download_post pass. status ∈ downloaded /
+    skipped_seen / skipped_disk / quarantined / error. `path` is the on-disk file
+    (downloaded / skipped_disk), the quarantine dest (quarantined), or None;
+    `error` is the failure/validation reason (error/quarantined) else None."""
+
+    media: object
+    status: str
+    path: Path | None
+    error: str | None
+
+
+@dataclass
+class PostRecordOutcome:
+    """Result of write_post_record — mirrors the MediaOutcome contract so the core
+    reports per-post handling. `path` is the _post.json sidecar (None when the post
+    had no id); the rest is the captured body's shape for the run log."""
+
+    path: Path | None
+    post_type: str | None
+    title: str | None
+    body_chars: int
+
+
+# -- base downloader (shared fetch/validate plumbing) ----------------------
+
+class BaseNativeDownloader:
+    """Shared download plumbing for native-platform downloaders: the streaming
+    GET (transient-retry + Range-resume) and file validation/quarantine. Platform
+    downloaders subclass this and implement `download_post` / `write_post_record`
+    / the per-media sidecar (and any platform-specific fetch, e.g. Patreon's
+    Mux/yt-dlp video branch). PURE: no DB; the seen-skip is an injected predicate.
+    """
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        platform: str,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        self.platform = platform
+        self._validate = validate
+        self._rate_limit = rate_limit or 0.0
+        self.session = session if session is not None else make_session(cookies_path)
+
+    # -- download seams ----------------------------------------------------
+
+    def _fetch_get(self, url: str, dest: Path) -> Path:
+        """Stream `url` to a .part then atomic-rename to `dest`."""
+        part = dest.with_name(dest.name + ".part")
+        try:
+            self._fetch_to_file(url, part)
+        except Exception:
+            with contextlib.suppress(OSError):
+                part.unlink()
+            raise
+        os.replace(part, dest)
+        return dest
+
+    def _fetch_to_file(self, url: str, dest: Path) -> None:
+        """Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
+        429 honoring Retry-After, 5xx) with backoff + resume-from-disk (Range);
+        failing fast on permanent 4xx (404/403). Resume: a retry with bytes on
+        disk asks `Range: bytes=<have>-`; 206 → append, 200 → restart clean, 416 →
+        already complete. The caller stages into a `.part` so a non-range server
+        never corrupts the output."""
+        attempt = 0
+        while True:
+            have = dest.stat().st_size if dest.exists() else 0
+            headers = {"Range": f"bytes={have}-"} if have > 0 else None
+            try:
+                resp = self.session.get(
+                    url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
+                )
+                if (resp.status_code == 429 or resp.status_code >= 500) \
+                        and attempt < _MAX_MEDIA_RETRIES:
+                    attempt += 1
+                    delay = retry_after_seconds(resp, attempt)
+                    log.warning(
+                        "%s media transient HTTP %d (%s) — backing off %.1fs "
+                        "(retry %d/%d)",
+                        self.platform, resp.status_code, url, delay, attempt,
+                        _MAX_MEDIA_RETRIES,
+                    )
+                    time.sleep(delay)
+                    continue
+                if have > 0 and resp.status_code == 416:
+                    return
+                resp.raise_for_status()
+                mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
+                with open(dest, mode) as fh:
+                    for chunk in resp.iter_content(chunk_size=_CHUNK):
+                        if chunk:
+                            fh.write(chunk)
+                return
+            except _TRANSIENT_TRANSPORT_EXC as exc:
+                if attempt >= _MAX_MEDIA_RETRIES:
+                    raise
+                attempt += 1
+                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
+                log.warning(
+                    "%s media transport error (%s) — backing off %.1fs "
+                    "(retry %d/%d): %s",
+                    self.platform, url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
+                )
+                time.sleep(delay)
+
+    # -- validation --------------------------------------------------------
+
+    def _validate_path(
+        self, path: Path, artist_slug: str, source_url: str | None = None
+    ) -> tuple[str | None, Path | None]:
+        """Validate a freshly-written file; quarantine if bad (shared
+        file_validator move + provenance sidecar). Returns (reason,
+        quarantine_dest) when quarantined, else (None, None). Logs the quarantine
+        so a corrupt file is visible in the worker logs, not just counted (#899
+        L2)."""
+        if not self._validate or not is_validatable(path):
+            return None, None
+        try:
+            result = validate_file(path)
+        except Exception as exc:
+            log.warning("Validator raised on %s: %s", path, exc)
+            return None, None
+        if result.ok:
+            return None, None
+        dest = quarantine_file(
+            self.images_root, path, artist_slug, self.platform,
+            url=source_url, result=result,
+        )
+        reason = result.reason or "validation failed"
+        log.warning(
+            "%s quarantined %s (%s) — %s",
+            self.platform, dest or path, artist_slug, reason,
+        )
+        return reason, (dest or path)
+
+    # -- sidecar (per-media, minimal) --------------------------------------
+
+    def _write_minimal_sidecar(
+        self, post: dict, media_path: Path, *, source_url: str | None = None
+    ) -> Path:
+        """Post-first per-media sidecar (#856): image identity ONLY
+        (category/id/source_url). The post body/links live solely in _post.json."""
+        data: dict = {"category": self.platform, "id": str(post.get("id") or "")}
+        if source_url:
+            data["source_url"] = source_url
+        sidecar_path = media_path.with_suffix(".json")
+        sidecar_path.write_text(json.dumps(data, indent=2))
+        return sidecar_path

backend/app/services/page_number_parser.py

@@ -0,0 +1,41 @@
+"""Parse a stated page number/range out of a post's title/description (FC-6.2).
+
+Artists often state where an installment sits in a series — "pages 9-12",
+"Page 5", "[3/8]". We use that to order chapters and flag missing-page gaps.
+This is best-effort: a confident match wins, otherwise we return None and the
+caller falls back to capture/post-date order. Keep it conservative — a wrong
+page number is worse than no page number — so matches require an explicit
+page keyword (page/pg/pp) or a bracketed N/M fraction, never a bare number.
+
+Supported forms (case-insensitive):
+  range    "pages 9-12", "pg 9–12", "pp. 9 - 12"        -> (9, 12)
+  fraction "page 3 of 8", "pg 3/8", "[3/8]", "(3/8)"     -> (3, 3)
+  single   "page 5", "pg 5", "pp 5"                       -> (5, 5)
+"""
+
+import re
+
+# Page keyword: page/pages/pg/pgs/pp/pp.  (NOT a bare "p" — too many false hits.)
+_KW = r"(?:pages?|pgs?|pp\.?)"
+_DASH = r"[-–—]"
+
+_RANGE = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\s*{_DASH}\s*(\d{{1,4}})", re.I)
+_OF = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\s*(?:of|/)\s*\d{{1,4}}\b", re.I)
+_BRACKET = re.compile(r"[\[(]\s*(\d{1,4})\s*/\s*\d{1,4}\s*[\])]")
+_SINGLE = re.compile(rf"\b{_KW}\s*(\d{{1,4}})\b", re.I)
+
+
+def parse_page_range(text: str | None) -> tuple[int, int] | None:
+    """Return (start, end) or None. start <= end; a single page yields (n, n)."""
+    if not text:
+        return None
+    m = _RANGE.search(text)
+    if m:
+        a, b = int(m.group(1)), int(m.group(2))
+        return (a, b) if a <= b else (b, a)
+    for rx in (_OF, _BRACKET, _SINGLE):
+        m = rx.search(text)
+        if m:
+            n = int(m.group(1))
+            return (n, n)
+    return None

backend/app/services/pagination.py

@@ -0,0 +1,31 @@
+"""Shared opaque keyset-pagination cursor.
+
+A cursor is base64(``<iso8601_sort_key>|<row_id>``). The sort key is whatever
+DESC-ordered timestamp a feed paginates on (gallery: effective_date; posts:
+COALESCE(post_date, downloaded_at); artists: created_at). `decode_cursor` rejects
+a malformed cursor with ValueError, which the API layer maps to HTTP 400.
+
+This was hand-rolled identically in gallery_service and post_feed_service (with
+artist_service importing gallery's copy). Two divergent copies of a cursor format
+silently break pagination in whichever feed drifts, so it lives once here now
+(DRY pattern sweep 2026-06-10).
+"""
+
+import base64
+from datetime import datetime
+
+CURSOR_SEPARATOR = "|"
+
+
+def encode_cursor(sort_key: datetime, row_id: int) -> str:
+    raw = f"{sort_key.isoformat()}{CURSOR_SEPARATOR}{row_id}"
+    return base64.urlsafe_b64encode(raw.encode()).decode()
+
+
+def decode_cursor(cursor: str) -> tuple[datetime, int]:
+    try:
+        raw = base64.urlsafe_b64decode(cursor.encode()).decode()
+        ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
+        return datetime.fromisoformat(ts_part), int(id_part)
+    except Exception as exc:
+        raise ValueError(f"invalid cursor: {cursor!r}") from exc

backend/app/services/patreon_client.py

@@ -0,0 +1,603 @@
+"""Native Patreon JSON:API client (build step 1 of the native ingester).
+
+Clean-room reimplementation of the Patreon `/api/posts` read path. This is a
+plain, synchronous client over `requests` — the orchestrator already wraps
+sync importer calls, so nothing here needs to be async (mirrors
+patreon_resolver.py, which wraps its sync lookup in run_in_executor at the
+call site).
+
+Scope (build step 1): fetch + page + parse only. This module is NOT wired into
+download_service yet — that is a later step. The public surface here exists so
+the later step can drive it:
+  - PatreonClient(cookies_path).iter_posts(campaign_id)
+        → (post, included_index, page_cursor)
+  - extract_media(post, included_index) → list[MediaItem]
+  - parse_cursor_from_url(url) → cursor
+
+Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we
+depend on (top-level `data`, media resources carrying `file_name`/`url`) are
+the contract. If a response comes back as an HTML login page or a media
+resource is missing the fields we resolve against, we raise PatreonDriftError
+rather than silently yielding empty media — so the later import step surfaces
+"Patreon changed something" instead of "creator has no posts".
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from html import unescape
+from pathlib import Path
+from urllib.parse import parse_qs, urlsplit
+
+import requests
+
+from ..utils.paths import filehash_from_url
+from ..utils.prosemirror import post_body_html
+from .native_ingest_common import (
+    _MAX_429_RETRIES,
+    NativeAuthError,
+    NativeDriftError,
+    NativeIngestError,
+    basename_from_url,
+    make_session,
+    retry_after_seconds,
+)
+
+log = logging.getLogger(__name__)
+
+_POSTS_URL = "https://www.patreon.com/api/posts"
+_TIMEOUT_SECONDS = 30.0
+
+# JSON:API request contract (observed from real traffic — see module plan).
+_INCLUDE = (
+    "campaign,access_rules,attachments,attachments_media,audio,images,media,"
+    "native_video_insights,user,user_defined_tags,ti_checks"
+)
+_FIELDS_POST = (
+    # `content` is the legacy flat-HTML body (Patreon now returns it null);
+    # `content_json_string` is the current ProseMirror-doc body. Request BOTH —
+    # post_body_html() prefers content (old posts) and falls back to converting
+    # content_json_string (current posts). #842.
+    "content,content_json_string,post_file,image,post_type,published_at,title,"
+    "url,patreon_url,current_user_can_view"
+)
+_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
+_FIELDS_CAMPAIGN = "name,url"
+
+# Inline post `content` is HTML; images are emitted as <img ... src="...">.
+# Pull every src; downstream dedup collapses any that duplicate a gallery item
+# by filehash. Tolerant of attribute ordering and single/double quotes.
+_CONTENT_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE)
+
+
+class PatreonAPIError(NativeIngestError):
+    """Base for native Patreon client failures. status_code / retry_after are
+    inherited from NativeIngestError (HTTP status; 429 Retry-After hint)."""
+
+
+class PatreonAuthError(PatreonAPIError, NativeAuthError):
+    """Authentication / authorization failure — missing or expired session
+    cookies, an insufficient pledge tier, or an HTML login/challenge page served
+    where JSON was expected. DISTINCT from drift: the fix is rotating the
+    credential, not updating the ingester. Maps to error_type 'auth_error'.
+    """
+
+
+class PatreonDriftError(PatreonAPIError, NativeDriftError):
+    """A JSON response did not match the JSON:API shape we depend on.
+
+    Raised for: a missing top-level `data` list, `data` not a list, or a media
+    resource lacking the `file_name`/`url` fields we resolve against. Fail loud
+    so the import step flags API drift (ingester needs update) instead of
+    silently importing nothing. An HTML-login / non-JSON body is auth, not
+    drift — that raises PatreonAuthError.
+    """
+
+
+@dataclass
+class MediaItem:
+    """One resolved downloadable item belonging to a post.
+
+    Fields:
+      url       — the CDN/download URL to fetch.
+      filename  — media `file_name` when present; otherwise the URL basename
+                  (NEVER a network call). Bounded/sane extension.
+      kind      — one of: "images", "image_large", "attachments", "postfile",
+                  "content". Mirrors gallery-dl's `files` content-type names so
+                  the later step can honor the same per-source content_types.
+      filehash  — the 32-char hex (MD5) segment from the CDN URL, or None if
+                  the URL carries no such segment. Used for in-post dedup and
+                  the cross-run seen-ledger.
+      post_id   — the owning post's id (so a flattened media list stays
+                  traceable to its post).
+    """
+
+    url: str
+    filename: str
+    kind: str
+    filehash: str | None
+    post_id: str
+
+
+def _filehash(url: str) -> str | None:
+    # Delegate to the shared extractor (utils.paths) so capture-time persistence
+    # and render-time inline-image matching use the EXACT same identity.
+    return filehash_from_url(url)
+
+
+def parse_cursor_from_url(url: str | None) -> str | None:
+    """Extract the `page[cursor]` query param from a links.next URL."""
+    if not url:
+        return None
+    query = urlsplit(url).query
+    values = parse_qs(query).get("page[cursor]")
+    if values and values[0]:
+        return values[0]
+    return None
+
+
+class PatreonClient:
+    """Synchronous Patreon JSON:API read client.
+
+    Construct with a path to a Netscape cookies.txt (the same file
+    CredentialService.get_cookies_path materializes). Cookies are loaded into a
+    requests.Session; no secure-context APIs are used.
+    """
+
+    def __init__(
+        self,
+        cookies_path: str | Path | None,
+        *,
+        request_sleep: float = 0.0,
+        max_retries: int = _MAX_429_RETRIES,
+    ):
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        self._session = make_session(cookies_path, accept="application/vnd.api+json")
+        # Politeness: seconds to sleep before each /api/posts page fetch (paces
+        # the rate-limited API endpoint). 0 = no pacing. plan #703.
+        self._request_sleep = request_sleep or 0.0
+        self._max_retries = max_retries
+
+    # -- request -----------------------------------------------------------
+
+    def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]:
+        params = {
+            "include": _INCLUDE,
+            "fields[post]": _FIELDS_POST,
+            "fields[media]": _FIELDS_MEDIA,
+            "fields[campaign]": _FIELDS_CAMPAIGN,
+            "filter[campaign_id]": campaign_id,
+            "filter[contains_exclusive_posts]": "true",
+            "filter[is_draft]": "false",
+            "sort": "-published_at",
+            "json-api-version": "1.0",
+        }
+        if cursor:
+            params["page[cursor]"] = cursor
+        return params
+
+    def _fetch(self, campaign_id: str, cursor: str | None) -> dict:
+        if self._request_sleep > 0:
+            time.sleep(self._request_sleep)  # pace the API endpoint
+        attempt = 0
+        while True:
+            try:
+                resp = self._session.get(
+                    _POSTS_URL,
+                    params=self._params(campaign_id, cursor),
+                    timeout=_TIMEOUT_SECONDS,
+                )
+            except requests.RequestException as exc:
+                raise PatreonAPIError(
+                    f"Patreon posts request failed (campaign_id={campaign_id}): {exc}"
+                ) from exc
+
+            # Transient rate-limit: back off and retry rather than failing the
+            # whole walk. Only a PERSISTENT 429 (retries exhausted) falls
+            # through to the terminal RATE_LIMITED raise below.
+            if resp.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
+                delay = retry_after_seconds(resp, attempt)
+                log.warning(
+                    "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)",
+                    campaign_id, delay, attempt, self._max_retries,
+                )
+                time.sleep(delay)
+                continue
+            break
+
+        if resp.status_code in (401, 403):
+            # Auth rejected — expired/missing cookies or an insufficient tier.
+            # Actionable as "rotate credentials", so it's auth, not drift/http.
+            raise PatreonAuthError(
+                f"Patreon posts API returned HTTP {resp.status_code} — auth "
+                f"rejected (cookies expired or tier insufficient; "
+                f"campaign_id={campaign_id})",
+                status_code=resp.status_code,
+            )
+        if resp.status_code != 200:
+            # A persistent 429 (retries exhausted) is terminal RATE_LIMITED — carry
+            # the server's raw Retry-After seconds so the cooldown matches its hint
+            # (plan #708 B1). Header is uncapped here; the cooldown clamps it.
+            retry_after = None
+            if resp.status_code == 429:
+                hdr = resp.headers.get("Retry-After")
+                if hdr:
+                    try:
+                        retry_after = float(hdr)
+                    except (TypeError, ValueError):
+                        retry_after = None
+            raise PatreonAPIError(
+                f"Patreon posts API returned HTTP {resp.status_code} "
+                f"(campaign_id={campaign_id})",
+                status_code=resp.status_code,
+                retry_after=retry_after,
+            )
+        try:
+            payload = resp.json()
+        except ValueError as exc:
+            # A non-JSON body here is almost always the HTML login/challenge
+            # page served when cookies are missing/expired — that is an AUTH
+            # failure (rotate cookies), not API drift (update the ingester) and
+            # not a transient network error.
+            raise PatreonAuthError(
+                "Patreon posts API returned a non-JSON response (likely an "
+                f"HTML login/challenge page — session expired; "
+                f"campaign_id={campaign_id}): {exc}"
+            ) from exc
+        return payload
+
+    # -- parsing -----------------------------------------------------------
+
+    @staticmethod
+    def _transform(response: dict) -> dict:
+        """Flatten the JSON:API `included` array for relationship resolution.
+
+        Returns a dict keyed by `(type, id)` → that resource's `attributes`
+        (so a post's relationships can be resolved in O(1)). Missing/oddly
+        shaped `included` entries are skipped rather than fatal — drift
+        detection for the top-level shape lives in _validate_response.
+        """
+        index: dict[tuple[str, str], dict] = {}
+        for inc in response.get("included") or []:
+            if not isinstance(inc, dict):
+                continue
+            rtype = inc.get("type")
+            rid = inc.get("id")
+            if rtype is None or rid is None:
+                continue
+            index[(str(rtype), str(rid))] = inc.get("attributes") or {}
+        return index
+
+    @staticmethod
+    def _validate_response(response: dict) -> None:
+        if not isinstance(response, dict):
+            raise PatreonDriftError("Patreon response was not a JSON object")
+        if "data" not in response:
+            raise PatreonDriftError("Patreon response missing top-level 'data' key")
+        data = response.get("data")
+        if not isinstance(data, list):
+            raise PatreonDriftError("Patreon response 'data' was not a list")
+
+    def _related_ids(self, post: dict, rel_name: str) -> list[str]:
+        rels = post.get("relationships") or {}
+        rel = rels.get(rel_name) or {}
+        data = rel.get("data")
+        if isinstance(data, dict):  # to-one relationship
+            data = [data]
+        if not isinstance(data, list):
+            return []
+        ids: list[str] = []
+        for ref in data:
+            if isinstance(ref, dict) and ref.get("id") is not None:
+                ids.append(str(ref["id"]))
+        return ids
+
+    @staticmethod
+    def _media_url(attrs: dict) -> str | None:
+        """Pick the best fetchable URL for a media resource.
+
+        Prefer the full-size `download_url`; fall back to the largest
+        `image_urls` size. gallery-dl prefers download_url too, only dipping
+        into image_urls when a smaller configured size is requested — FC
+        always wants the original, so download_url first.
+        """
+        download_url = attrs.get("download_url")
+        if isinstance(download_url, str) and download_url:
+            return download_url
+        image_urls = attrs.get("image_urls")
+        if isinstance(image_urls, dict):
+            for key in ("original", "full", "large", "default"):
+                candidate = image_urls.get(key)
+                if isinstance(candidate, str) and candidate:
+                    return candidate
+            # Otherwise take any non-empty string value.
+            for candidate in image_urls.values():
+                if isinstance(candidate, str) and candidate:
+                    return candidate
+        return None
+
+    def _media_item(self, attrs: dict, kind: str, post_id: str) -> MediaItem:
+        url = self._media_url(attrs)
+        if not url:
+            raise PatreonDriftError(
+                f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL "
+                f"(no download_url / image_urls)"
+            )
+        # file_name is OPTIONAL: Patreon legitimately serves some gallery images
+        # without it (operator-flagged 2026-06-07, BlenderKnight post 73665615),
+        # and the URL basename is a fine fallback — the same thing gallery-dl
+        # uses. A genuine schema change shows up as no URL (above) or a media id
+        # absent from `included` (caller), not a missing name.
+        file_name = attrs.get("file_name")
+        filename = file_name if isinstance(file_name, str) and file_name else basename_from_url(url)
+        return MediaItem(
+            url=url,
+            filename=filename,
+            kind=kind,
+            filehash=_filehash(url),
+            post_id=post_id,
+        )
+
+    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
+        """Resolve all downloadable media for one post.
+
+        Walks the kinds in the same order gallery-dl does — images,
+        image_large (post cover), attachments, postfile, content (inline
+        <img>) — and dedups within the post by filehash (first wins). The
+        image_large cover commonly duplicates a gallery image; deduping by
+        filehash collapses them to the gallery item (encountered first).
+        """
+        post_id = str(post.get("id") or "")
+        attrs = post.get("attributes") or {}
+        items: list[MediaItem] = []
+
+        def _resolve_rel(rel_name: str, kind: str) -> None:
+            for mid in self._related_ids(post, rel_name):
+                media_attrs = included_index.get(("media", mid))
+                if media_attrs is None:
+                    # Referenced but not in `included`: a media id with no
+                    # resource is drift (we asked for include=media).
+                    raise PatreonDriftError(
+                        f"Patreon post {post_id} references media {mid} "
+                        f"({rel_name}) not present in 'included'"
+                    )
+                items.append(self._media_item(media_attrs, kind, post_id))
+
+        # 1. gallery images
+        _resolve_rel("images", "images")
+
+        # 2. image_large — the post-level cover (`image.large_url`). Not a
+        #    media relationship; it lives on the post attributes.
+        image = attrs.get("image")
+        if isinstance(image, dict):
+            large_url = image.get("large_url") or image.get("url")
+            if isinstance(large_url, str) and large_url:
+                items.append(
+                    MediaItem(
+                        url=large_url,
+                        filename=basename_from_url(large_url),
+                        kind="image_large",
+                        filehash=_filehash(large_url),
+                        post_id=post_id,
+                    )
+                )
+
+        # 3. attachments
+        _resolve_rel("attachments_media", "attachments")
+
+        # 4. postfile — the post's primary attached file (`post_file`).
+        post_file = attrs.get("post_file")
+        if isinstance(post_file, dict):
+            pf_url = post_file.get("url") or post_file.get("download_url")
+            if isinstance(pf_url, str) and pf_url:
+                pf_name = post_file.get("name")
+                filename = (
+                    pf_name
+                    if isinstance(pf_name, str) and pf_name
+                    else basename_from_url(pf_url)
+                )
+                items.append(
+                    MediaItem(
+                        url=pf_url,
+                        filename=filename,
+                        kind="postfile",
+                        filehash=_filehash(pf_url),
+                        post_id=post_id,
+                    )
+                )
+
+        # 5. content — inline <img> in the post HTML body.
+        content = attrs.get("content")
+        if isinstance(content, str) and content:
+            for raw_src in _CONTENT_IMG_RE.findall(content):
+                src = unescape(raw_src)
+                if not src:
+                    continue
+                items.append(
+                    MediaItem(
+                        url=src,
+                        filename=basename_from_url(src),
+                        kind="content",
+                        filehash=_filehash(src),
+                        post_id=post_id,
+                    )
+                )
+
+        return _dedup_by_filehash(items)
+
+    @staticmethod
+    def post_meta(post: dict) -> dict:
+        """Title + published date for a post — for the preview sample (plan #708
+        B4). Part of the client contract `ingest_core.Ingester.preview` calls."""
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title")
+        published = attrs.get("published_at")
+        return {
+            "title": title if isinstance(title, str) else None,
+            "date": published if isinstance(published, str) else None,
+        }
+
+    @staticmethod
+    def post_is_gated(post: dict) -> bool:
+        """True when the authenticated account CANNOT view this post's content
+        (#874). Patreon serves only BLURRED locked-preview thumbnails for
+        paywalled / insufficient-tier posts, and `current_user_can_view` on the
+        post attributes is the access flag (it IS in `_FIELDS_POST`). The walk
+        skips a gated post ENTIRELY — no media, no post-record stub — so those
+        unusable previews never get downloaded.
+
+        Gate ONLY on an explicit `current_user_can_view == False`. A missing /
+        None flag (older posts, a sparse fieldset, or API drift) is treated as
+        viewable, so we never over-filter accessible posts on an absent field.
+        Part of the client contract the core consumes via getattr — an optional
+        seam, so stub clients / not-yet-migrated platforms simply never gate."""
+        attrs = post.get("attributes") or {}
+        return attrs.get("current_user_can_view") is False
+
+    @staticmethod
+    def post_record_key(post: dict) -> tuple[str, str] | None:
+        """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or
+        None when the post has no id. The synthetic `post:<id>` key lets the
+        generic core gate text-post capture through the SAME seen-ledger as media
+        — so a text post's body is detail-fetched + recorded ONCE, not re-fetched
+        every tick. Part of the client contract the core uses via getattr."""
+        pid = post.get("id")
+        pid = str(pid) if pid is not None else ""
+        if not pid:
+            return None
+        return (f"post:{pid}", pid)
+
+    # -- iteration ---------------------------------------------------------
+
+    def iter_posts(
+        self, campaign_id: str, cursor: str | None = None
+    ) -> Iterator[tuple[dict, dict, str | None]]:
+        """Yield (post, included_index, page_cursor) for every post in the feed.
+
+        Pages newest→oldest via `links.next`, validating each response for
+        drift before yielding. The triple gives a caller everything it needs to
+        resolve and checkpoint without re-fetching:
+          - post           — the raw post resource.
+          - included_index — the page's flattened `included` (the same object
+                             for every post on a page), to pass straight to
+                             extract_media(post, included_index).
+          - page_cursor    — the cursor that FETCHED this post's page (None for
+                             the first page). The caller checkpoints THIS value,
+                             matching the existing backfill cursor logic where
+                             the saved cursor re-fetches the page being
+                             processed (so a chunk cut mid-page resumes the page,
+                             not the one after it).
+        """
+        current_cursor = cursor
+        while True:
+            response = self._fetch(campaign_id, current_cursor)
+            self._validate_response(response)
+            page_cursor = current_cursor
+            included_index = self._transform(response)
+            for post in response.get("data") or []:
+                if isinstance(post, dict):
+                    yield post, included_index, page_cursor
+            next_url = (response.get("links") or {}).get("next")
+            next_cursor = parse_cursor_from_url(next_url)
+            if not next_cursor:
+                return
+            current_cursor = next_cursor
+
+    # -- detail (full body enrichment) -------------------------------------
+
+    def fetch_post_detail_content(self, post_id: str) -> str | None:
+        """Best-effort fetch of a post's body (as HTML) from the per-post DETAIL
+        endpoint (`/api/posts/{id}`).
+
+        Patreon deprecated the flat `content` HTML field — it returns null on the
+        feed AND the detail endpoint, for every post type. The real body now lives
+        in `content_json_string` (a ProseMirror doc), and is only returned under
+        the DEFAULT post fieldset: a sparse `fields[post]=content` request OMITS
+        it (confirmed against the live API 2026-06-15). So we request the default
+        fieldset (no `fields[post]`) and resolve the body via `post_body_html`
+        (content → else convert content_json_string). The downloader calls this to
+        enrich a post whose feed body was empty before writing the sidecar.
+
+        Best-effort BY DESIGN: a body we can't fetch must never fail the walk.
+        Every failure path returns None rather than raising — distinct from the
+        loud drift/auth raises on the feed path, which gate real downloads.
+        """
+        if not post_id:
+            return None
+        if self._request_sleep > 0:
+            time.sleep(self._request_sleep)  # pace the API endpoint (plan #703)
+        # No `fields[post]` — the default fieldset is the only shape that returns
+        # content_json_string (the body). A sparse fieldset nulls it out.
+        try:
+            resp = self._session.get(f"{_POSTS_URL}/{post_id}", timeout=_TIMEOUT_SECONDS)
+        except requests.RequestException as exc:
+            log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
+            return None
+        if resp.status_code != 200:
+            log.warning(
+                "Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id
+            )
+            return None
+        try:
+            payload = resp.json()
+        except ValueError:
+            return None
+        data = payload.get("data") if isinstance(payload, dict) else None
+        attrs = data.get("attributes") if isinstance(data, dict) else None
+        body = post_body_html(attrs)
+        if body and body.strip():
+            log.info("post-detail: fetched %d chars (post %s)", len(body), post_id)
+            return body
+        ptype = attrs.get("post_type") if isinstance(attrs, dict) else None
+        log.info(
+            "post-detail: no body (post %s, post_type=%s)", post_id, ptype,
+        )
+        return None
+
+    # -- verify ------------------------------------------------------------
+
+    def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
+        """Cheap auth probe: fetch the first `/api/posts` page and report whether
+        the credential authenticated, WITHOUT downloading anything.
+
+        Returns `(ok, message)` matching the credential-verify contract:
+          - True  — authenticated (the feed returned a valid JSON:API page).
+          - False — the credential was rejected (PatreonAuthError: 401/403, or an
+                    HTML login page → cookies expired / tier insufficient).
+          - None  — inconclusive: API drift (our parser is stale, not a cred
+                    problem) or a transient network/HTTP error.
+        """
+        try:
+            response = self._fetch(campaign_id, None)
+            self._validate_response(response)
+        except PatreonAuthError as exc:
+            return False, f"Patreon rejected the credential — {exc}"
+        except PatreonDriftError as exc:
+            return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
+        except PatreonAPIError as exc:
+            return None, f"Couldn't verify (network/HTTP issue): {exc}"
+        return True, "Credentials valid — the Patreon feed authenticated."
+
+
+def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
+    """Drop later items sharing a filehash with an earlier one (first wins).
+
+    Items with no filehash (None) are never deduped against each other — we
+    can't prove they're the same file, so keep them all.
+    """
+    seen: set[str] = set()
+    out: list[MediaItem] = []
+    for item in items:
+        if item.filehash is not None:
+            if item.filehash in seen:
+                continue
+            seen.add(item.filehash)
+        out.append(item)
+    return out

backend/app/services/patreon_downloader.py

@@ -0,0 +1,418 @@
+"""Native Patreon media downloader (build step 2b of the native ingester).
+
+Given a Patreon post and its already-resolved `MediaItem`s (from
+patreon_client.extract_media), download the media to the EXACT on-disk layout
+gallery-dl produces, write a sidecar JSON the existing importer consumes, and
+report per-media outcomes.
+
+This module is PURE: no DB. The cross-run seen-ledger and the iter_posts
+orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate
+(`is_seen`), so this module needs no DB and is unit-testable without network.
+
+On-disk layout (matches gallery-dl):
+  <images_root>/<artist_slug>/patreon/<DIR>/<NN>_<filename>
+  where <DIR> = "<YYYY-MM-DD>_<post_id>_<title40>" (date prefix omitted when the
+  post's published_at is unparseable), <NN> is the 1-based index of the item in
+  the post zero-padded to 2, and <filename> is MediaItem.filename. The sidecar
+  is written next to the media as <NN>_<stem>.json (media_path.with_suffix).
+
+Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path
+endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same
+Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py
+_get_default_config). yt-dlp may remux to a container of its own choosing, so we
+accept the actual output extension and record the real path.
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import time
+from collections.abc import Callable
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import requests
+
+from ..utils.prosemirror import post_body_html
+from .native_ingest_common import (
+    _BACKOFF_CAP_SECONDS,
+    _MAX_MEDIA_RETRIES,
+    BaseNativeDownloader,
+    MediaOutcome,
+    PostRecordOutcome,
+    post_dir_name,
+    sanitize_segment,
+)
+
+log = logging.getLogger(__name__)
+
+# yt-dlp subprocess wall-clock per attempt (video only; the shared HTTP fetch
+# budgets live in BaseNativeDownloader).
+_TIMEOUT_SECONDS = 120.0
+
+# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT
+# playback policy checks Referer/Origin on every request, so yt-dlp must send
+# Patreon's, not its own default. (gallery-dl forwarded the same headers before
+# the #697 cutover removed its Patreon path; this is now the only place they
+# live.)
+_VIDEO_HEADERS = {
+    "Referer": "https://www.patreon.com/",
+    "Origin": "https://www.patreon.com",
+}
+
+
+def _is_video_url(url: str) -> bool:
+    parts = urlsplit(url)
+    if parts.hostname and parts.hostname.lower() == "stream.mux.com":
+        return True
+    return parts.path.lower().endswith(".m3u8")
+
+
+class PatreonDownloader(BaseNativeDownloader):
+    """Download resolved Patreon media to gallery-dl's on-disk layout. Subclasses
+    BaseNativeDownloader for the shared streaming GET (transient-retry +
+    Range-resume) and validation/quarantine; adds the Mux/HLS yt-dlp video branch
+    and the detail-fetch body enrichment. PURE: no DB. `_run_ytdlp` is
+    monkeypatchable and the HTTP session is the injectable `session=` seam.
+    """
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+        content_fetcher: Callable[[str], str | None] | None = None,
+    ):
+        super().__init__(
+            images_root, cookies_path, platform="patreon",
+            validate=validate, rate_limit=rate_limit, session=session,
+        )
+        # Best-effort enrichment seam: (post_id) -> full HTML body, or None. The
+        # feed endpoint often omits `content`; the adapter wires this to
+        # PatreonClient.fetch_post_detail_content so the sidecar captures the
+        # real body (formatting + inline <img> + external <a href> links).
+        # None in unit tests / when enrichment isn't wanted.
+        self._content_fetcher = content_fetcher
+
+    # -- public ------------------------------------------------------------
+
+    def download_post(
+        self,
+        post: dict,
+        media_items: list,
+        artist_slug: str,
+        *,
+        is_seen: Callable[[object], bool] = lambda m: False,
+        should_stop: Callable[[], bool] = lambda: False,
+        recapture: bool = False,
+    ) -> list[MediaOutcome]:
+        """Download every media item of one post; return per-item outcomes.
+
+        Builds the post directory, iterates media (1-based NN), applies the
+        two-tier skip (injected is_seen, then disk), downloads (plain GET or
+        yt-dlp for video), writes the sidecar for each freshly-downloaded item,
+        validates, and returns outcomes. Resilient: one media's failure yields
+        an "error" outcome for that item; the rest proceed.
+
+        `should_stop()` is polled BEFORE each media item: a media-dense post can
+        otherwise run a backfill chunk far past its time-box (the engine only
+        re-checks the budget between posts), so we honour the deadline mid-post
+        and return the items done so far — the rest re-fetch next chunk (they
+        were never marked seen). Bounds chunk overrun to one media download.
+
+        `recapture` (#830): don't re-download already-present media, but DO surface
+        on-disk media as `skipped_disk` (with its path) even when the seen-ledger
+        would tier-1 skip it — so the engine can backfill source_filehash for
+        inline-image localization. Genuinely-missing seen media is NOT refetched.
+        """
+        post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
+        outcomes: list[MediaOutcome] = []
+
+        for i, media in enumerate(media_items, start=1):
+            if should_stop():
+                break
+            try:
+                outcomes.append(
+                    self._download_one(
+                        post, media, post_dir, artist_slug, i, is_seen,
+                        recapture=recapture,
+                    )
+                )
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "Patreon media failed (post %s, item %d): %s",
+                    post.get("id"), i, exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    # -- per-item ----------------------------------------------------------
+
+    def _download_one(
+        self,
+        post: dict,
+        media,
+        post_dir: Path,
+        artist_slug: str,
+        index: int,
+        is_seen: Callable[[object], bool],
+        *,
+        recapture: bool = False,
+    ) -> MediaOutcome:
+        # tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
+        # short-circuit here — we fall through to the disk check so an on-disk
+        # seen file is surfaced as skipped_disk (with its path) for source_filehash
+        # backfill; a seen file that's NOT on disk is left alone (not refetched).
+        seen = is_seen(media)
+        if seen and not recapture:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        nn = f"{index:02d}"
+        final_name = sanitize_segment(f"{nn}_{media.filename}")
+        media_path = post_dir / final_name
+
+        # tier-2: already on disk.
+        if media_path.exists():
+            return MediaOutcome(
+                media=media, status="skipped_disk", path=media_path, error=None
+            )
+        # Video may land at a different extension; honor a pre-existing remux.
+        if _is_video_url(media.url):
+            existing = self._existing_video_output(media_path)
+            if existing is not None:
+                return MediaOutcome(
+                    media=media, status="skipped_disk", path=existing, error=None
+                )
+
+        # recapture: a seen item that isn't on disk is NOT re-downloaded (that's
+        # recovery's job) — recapture only re-grabs post text + localizes existing
+        # files. Returns skipped_seen so the run-of-seen / counts stay consistent.
+        if seen:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        post_dir.mkdir(parents=True, exist_ok=True)
+
+        # Pace real downloads only (the skips above already returned). plan #703.
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+
+        if _is_video_url(media.url):
+            out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
+            if out_path is None or not Path(out_path).exists():
+                return MediaOutcome(
+                    media=media,
+                    status="error",
+                    path=None,
+                    error="yt-dlp produced no output",
+                )
+            out_path = Path(out_path)
+        else:
+            out_path = self._fetch_get(media.url, media_path)
+            reason, quarantine_dest = self._validate_path(
+                out_path, artist_slug, media.url
+            )
+            if reason is not None:
+                # Quarantined (corrupt/invalid) — distinct from a download error
+                # so the run can report a real files_quarantined count + paths.
+                return MediaOutcome(
+                    media=media, status="quarantined",
+                    path=quarantine_dest, error=reason,
+                )
+
+        self._write_sidecar(post, out_path, source_url=media.url)
+        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
+
+    # -- video (Mux/HLS via yt-dlp) ----------------------------------------
+    # The plain-GET streaming path (_fetch_get / _fetch_to_file) and
+    # _validate_path are inherited from BaseNativeDownloader.
+
+    def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
+        """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
+
+        Returns the actual output path (yt-dlp may remux to a different
+        container, so we resolve the real file afterward). Overridden/
+        monkeypatched in tests to avoid spawning a real subprocess.
+
+        The output template uses `dest` without its extension; yt-dlp appends
+        the chosen container extension. We pass Referer/Origin (Mux JWT policy)
+        and the cookies file.
+
+        Mirrors the GET path's transient/permanent split (plan #705 #8): a hung
+        fetch (TimeoutExpired) or a spawn failure (OSError) is TRANSIENT — back
+        off and retry. A non-zero yt-dlp exit (CalledProcessError) is treated as
+        PERMANENT for this pass — yt-dlp already does its OWN internal network
+        retries, so a non-zero exit is effectively a real failure (private/gone/
+        geo-blocked), like a 4xx on the GET path: fail fast to the per-item error
+        → dead-letter path.
+        """
+        dest = Path(dest)
+        out_template = str(dest.with_suffix("")) + ".%(ext)s"
+        cmd = ["yt-dlp", "--no-progress", "-o", out_template]
+        for key, value in headers.items():
+            cmd += ["--add-header", f"{key}:{value}"]
+        if self.cookies_path and os.path.isfile(self.cookies_path):
+            cmd += ["--cookies", self.cookies_path]
+        cmd.append(url)
+        attempt = 0
+        while True:
+            try:
+                subprocess.run(
+                    cmd,
+                    check=True,
+                    capture_output=True,
+                    text=True,
+                    timeout=_TIMEOUT_SECONDS,
+                )
+                break
+            except subprocess.CalledProcessError as exc:
+                # Permanent for this pass — fail fast (no retry).
+                log.warning(
+                    "yt-dlp failed (exit %s) for %s: %s",
+                    exc.returncode, url, (exc.stderr or "").strip() or exc,
+                )
+                return None
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                # Transient — back off and retry, like a transport blip on a GET.
+                if attempt >= _MAX_MEDIA_RETRIES:
+                    log.warning(
+                        "yt-dlp transient failure exhausted for %s: %s", url, exc
+                    )
+                    return None
+                attempt += 1
+                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
+                log.warning(
+                    "yt-dlp transient failure (%s) — backing off %.1fs "
+                    "(retry %d/%d): %s",
+                    url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
+                )
+                time.sleep(delay)
+        return self._existing_video_output(dest)
+
+    def _existing_video_output(self, dest: Path) -> Path | None:
+        """Find a yt-dlp output for `dest` regardless of chosen extension.
+
+        Returns `dest` itself if present, else any sibling sharing the same
+        stem (the remuxed container). None if nothing matched.
+        """
+        dest = Path(dest)
+        if dest.exists():
+            return dest
+        stem = dest.with_suffix("").name
+        parent = dest.parent
+        if not parent.is_dir():
+            return None
+        for cand in sorted(parent.iterdir()):
+            if cand.is_file() and cand.stem == stem and cand.name != dest.name:
+                return cand
+        return None
+
+    # -- sidecar -----------------------------------------------------------
+
+    def _write_sidecar(
+        self, post: dict, media_path: Path, *, source_url: str | None = None
+    ) -> Path:
+        """Write the per-media sidecar next to `media_path` — post-first (#856).
+
+        On the native ingester the POST-RECORD (`write_post_record` → `_post.json`)
+        is the sole writer of the post body/links/metadata, captured once per post
+        BEFORE its media in the walk. So the per-media sidecar carries ONLY
+        image-specific identity: `category` (platform) + `id` (external_post_id, to
+        link provenance to the right Post) + this file's `source_url` (its CDN URL,
+        #830 Phase 2 — the importer persists its filehash so the body's inline
+        `<img src>` remaps to the local copy at render time). No body: writing it
+        next to every image duplicated the post body N+1× and risked divergence
+        (milestone #67). The importer skips post fields for these (post_first).
+        """
+        return self._write_sidecar_data(
+            post, media_path.with_suffix(".json"), source_url=source_url,
+            minimal=True,
+        )
+
+    def _write_sidecar_data(
+        self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
+        minimal: bool = False,
+    ) -> Path:
+        """Serialize the post's metadata to `sidecar_path`. The post-only record
+        (`write_post_record`) writes the FULL post (body/title/date/url); the
+        per-media sidecar (`_write_sidecar`, minimal=True) writes only image
+        identity (category/id/source_url) — post-first (#856). `source_url` is set
+        only for the per-media sidecar — a media-less post has no source file."""
+        if minimal:
+            data = {"category": "patreon", "id": str(post.get("id") or "")}
+            if source_url:
+                data["source_url"] = source_url
+            sidecar_path.write_text(json.dumps(data, indent=2))
+            return sidecar_path
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title")
+        # Resolve the body HTML from the feed attrs: legacy flat `content`, else
+        # convert the current `content_json_string` ProseMirror doc (#842).
+        content = post_body_html(attrs)
+        # The feed/list endpoint frequently returns an empty body; the full body
+        # only comes from the per-post detail endpoint. Enrich on first write for
+        # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post`
+        # dict — so a multi-image post fetches detail at most once, the post-record
+        # body-length read reuses it, and a fully-seen post (no fresh download → no
+        # sidecar write) never pays the extra GET.
+        if (not content or not content.strip()) and self._content_fetcher:
+            fetched = self._content_fetcher(str(post.get("id") or ""))
+            if fetched:
+                content = fetched
+        if isinstance(content, str) and content.strip():
+            attrs["content"] = content
+            post["attributes"] = attrs
+        published = attrs.get("published_at")
+        url = attrs.get("url")
+        data = {
+            "category": "patreon",
+            "id": str(post.get("id") or ""),
+            "title": title if isinstance(title, str) else "",
+            "content": content if isinstance(content, str) else "",
+            "published_at": published if isinstance(published, str) else None,
+            "url": url if isinstance(url, str) else None,
+        }
+        if source_url:
+            data["source_url"] = source_url
+        sidecar_path.write_text(json.dumps(data, indent=2))
+        return sidecar_path
+
+    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
+        """Write a post-ONLY sidecar (no media file) for a media-less post, so
+        the importer can still upsert the Post + its body — text posts often hold
+        the only copy of an external <a href> link. Named `_post.json`: the
+        leading underscore keeps it from colliding with a media sidecar
+        (`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
+        by find_sidecar.
+
+        Returns a PostRecordOutcome (path None when the post has no id) carrying
+        the captured body's shape — post_type + final char count — so the engine
+        can log per-post handling without re-reading the post itself.
+        """
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
+        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
+        pid = str(post.get("id") or "")
+        if not pid:
+            return PostRecordOutcome(
+                path=None, post_type=post_type, title=title, body_chars=0,
+            )
+        post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
+        post_dir.mkdir(parents=True, exist_ok=True)
+        path = self._write_sidecar_data(post, post_dir / "_post.json")
+        # _write_sidecar_data has by now memoized any detail-fetched body onto
+        # post["attributes"]["content"], so re-read it for the FINAL char count.
+        body = (post.get("attributes") or {}).get("content")
+        body_chars = len(body) if isinstance(body, str) else 0
+        return PostRecordOutcome(
+            path=path, post_type=post_type, title=title, body_chars=body_chars,
+        )

backend/app/services/patreon_ingester.py

@@ -0,0 +1,153 @@
+"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core.
+
+The orchestration that drives a native subscription walk (page a feed → extract
+media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor
+→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery)
+now lives in `ingest_core.Ingester` — it's identical for every platform. This
+module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
+ledger models/constraints/key into the core and supplies the Patreon-specific
+failure mapping. `download_service.download_source` calls `PatreonIngester.run`
+exactly as before; the public surface (this class, `_ledger_key`,
+`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.
+
+Three modes (selected by `download_service` from `config_overrides` state):
+  - tick      — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
+                after N contiguous already-have-it items (the cheap native
+                equivalent of gallery-dl's `exit:20`, now free of per-file HEADs).
+  - backfill  — full-history walk in a time-boxed chunk, resuming from the
+                pagination cursor checkpoint; reaches the bottom → "complete".
+  - recovery  — like backfill but BYPASSES the tier-1 seen-ledger AND the
+                dead-letter ledger, so deliberately-dropped-and-deleted near-dups
+                get re-fetched and re-evaluated under the current pHash threshold
+                (tier-2 disk skip still spares files we kept).
+
+The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` /
+`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch
+— never held across a network fetch ([[db-connection-held-across-subprocess]]).
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from ..models import PatreonFailedMedia, PatreonSeenMedia
+from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
+from .patreon_client import MediaItem, PatreonAPIError, PatreonClient
+from .patreon_downloader import PatreonDownloader
+from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
+
+__all__ = [
+    "DEAD_LETTER_THRESHOLD",
+    "PatreonIngester",
+    "_ledger_key",
+    "verify_patreon_credential",
+]
+
+log = logging.getLogger(__name__)
+
+# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
+# synthesized key so a pathologically long file_name can't overflow the column.
+_LEDGER_KEY_MAX = 128
+
+
+def _ledger_key(media: MediaItem) -> str:
+    """Stable per-media identity for the cross-run seen-ledger.
+
+    A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
+    natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
+    content hash at discovery) and the odd inline-content `<img>` pointing at a
+    hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
+    sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
+    stable proxy. Bounded to the column width.
+    """
+    if media.filehash:
+        return media.filehash
+    return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
+
+
+class PatreonIngester(Ingester):
+    """Walk a Patreon campaign's posts, download unseen media, return a
+    `DownloadResult`. A thin adapter over `ingest_core.Ingester`.
+
+    Construct with the per-source `cookies_path` and a sync sessionmaker for the
+    ledgers. `client` / `downloader` are injectable seams so unit tests run
+    without network, subprocess, or a real CDN.
+    """
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None,
+        session_factory: Callable[[], object],
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        request_sleep: float = 0.0,
+        client: PatreonClient | None = None,
+        downloader: PatreonDownloader | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        # Pacing (plan #703): request_sleep paces the API page fetches,
+        # rate_limit paces the media downloads. Injected client/downloader (in
+        # tests) already carry their own pacing, so these only apply to the
+        # default-constructed ones.
+        resolved_client = (
+            client
+            if client is not None
+            else PatreonClient(cookies_path, request_sleep=request_sleep)
+        )
+        resolved_downloader = (
+            downloader
+            if downloader is not None
+            else PatreonDownloader(
+                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
+                # Enrich empty feed bodies from the per-post detail endpoint, via
+                # the SAME client (shares its cookie session + request pacing).
+                content_fetcher=resolved_client.fetch_post_detail_content,
+            )
+        )
+        super().__init__(
+            client=resolved_client,
+            downloader=resolved_downloader,
+            session_factory=session_factory,
+            seen_model=PatreonSeenMedia,
+            failed_model=PatreonFailedMedia,
+            seen_constraint="uq_patreon_seen_media_source_id",
+            failed_constraint="uq_patreon_failed_media_source_id",
+            ledger_key=_ledger_key,
+            platform="patreon",
+            error_base=PatreonAPIError,
+            # API_DRIFT message phrasing; the base Ingester._failure_result owns
+            # the auth/drift/HTTP→error_type mapping now (shared across platforms).
+            drift_label="Patreon API",
+        )
+
+
+async def verify_patreon_credential(
+    url: str,
+    cookies_path: str | None,
+    overrides: dict | None,
+) -> tuple[bool | None, str]:
+    """Native Patreon credential probe — the verify counterpart to the ingester's
+    download path, sharing its campaign-id resolution. Resolves the campaign id
+    (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
+    via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
+    (True / False / None) so download_backends.verify_credential can treat it
+    interchangeably with the gallery-dl probe. No download, no DB.
+    """
+    campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
+    if not campaign_id:
+        vanity = extract_vanity(url)
+        return None, (
+            f"Couldn't resolve the Patreon campaign id — can't verify. "
+            f"source_url={url!r}; vanity={vanity!r} "
+            "(cookies expired, or the creator moved/renamed?)."
+        )
+    client = PatreonClient(cookies_path)
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, client.verify_auth, campaign_id)

backend/app/services/patreon_resolver.py

@@ -19,18 +19,56 @@ import asyncio
 import http.cookiejar
 import logging
 import os
+import re
 
 import requests
 
 log = logging.getLogger(__name__)
 
 _CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
+_POSTS_API = "https://www.patreon.com/api/posts"
+
+# A source URL of the form `.../id:<digits>` already carries the campaign id
+# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the
+# two paths don't overlap.
+#
+# Patreon serves the creator vanity under several path prefixes — bare
+# (`patreon.com/Atole`), `c/` (`patreon.com/c/Atole`), and `cw/`
+# (`patreon.com/cw/Atole`, its current "creator workspace" URL). The optional
+# prefix group must list `cw/` BEFORE `c/` so the longer prefix wins — otherwise
+# `cw/Atole` matches the bare branch and yields vanity="cw" (operator-flagged
+# 2026-06-07: every `/cw/` source failed resolution on vanity="cw").
+_ID_URL_RE = re.compile(r"/id:(\d+)")
+# A single-post permalink — patreon.com/posts/<slug>-<post_id> (or bare
+# /posts/<post_id>). The trailing digits are the post id; the creator's
+# campaign is resolved from the post itself (operator-flagged 2026-06-07: a
+# /posts/ source resolved vanity="posts"). `posts/` is excluded from the vanity
+# regex so it never masquerades as a creator slug.
+_POST_URL_RE = re.compile(r"/posts/(?:[^/?#]*-)?(\d+)(?:[/?#]|$)")
+_VANITY_PREFIXES = ("cw/", "c/")
+_VANITY_RE = re.compile(
+    r"^https?://(?:www\.)?patreon\.com/(?:cw/|c/)?(?!(?:id:|posts/))([^/?#]+)",
+    re.IGNORECASE,
+)
 _USER_AGENT = (
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
 )
 _TIMEOUT_SECONDS = 10.0
 
+# Fallback resolution: Patreon's `/api/campaigns?filter[vanity]=` lookup has
+# proven unreliable (returns empty `data` for creators that clearly exist —
+# operator-flagged 2026-06-06). gallery-dl never used that endpoint; it scrapes
+# the campaign id out of the creator page's bootstrap JSON. We do the same as a
+# fallback: fetch the creator page HTML and pull the first campaign id out of
+# any of these embeddings (ordered most- to least-specific).
+_PAGE_CAMPAIGN_ID_PATTERNS = (
+    re.compile(r'"id":\s*"(\d+)",\s*"type":\s*"campaign"'),
+    re.compile(r'"campaign":\s*\{\s*"data":\s*\{\s*"id":\s*"(\d+)"'),
+    re.compile(r"/api/campaigns/(\d+)"),
+    re.compile(r'"campaign_id":\s*"?(\d+)'),
+)
+
 
 def _load_cookie_jar(cookies_path: str | None) -> http.cookiejar.MozillaCookieJar | None:
     if not cookies_path or not os.path.isfile(cookies_path):
@@ -45,6 +83,15 @@ def _load_cookie_jar(cookies_path: str | None) -> http.cookiejar.MozillaCookieJa
 
 
 def _sync_lookup(vanity: str, cookies_path: str | None) -> str | None:
+    """Resolve a vanity to a campaign id: try the campaigns API first (cheap,
+    structured), then fall back to scraping the creator page (robust against the
+    API's empty-data failures). Returns None only when both miss."""
+    return _lookup_via_api(vanity, cookies_path) or _lookup_via_page(
+        vanity, cookies_path
+    )
+
+
+def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
     jar = _load_cookie_jar(cookies_path)
     headers = {
         "User-Agent": _USER_AGENT,
@@ -92,6 +139,87 @@ def _sync_lookup(vanity: str, cookies_path: str | None) -> str | None:
     return campaign_id
 
 
+def _scrape_campaign_id(html: str) -> str | None:
+    """First campaign id found in creator-page HTML via the known embeddings."""
+    if not isinstance(html, str):
+        return None
+    for pat in _PAGE_CAMPAIGN_ID_PATTERNS:
+        m = pat.search(html)
+        if m:
+            return m.group(1)
+    return None
+
+
+def _lookup_via_page(vanity: str, cookies_path: str | None) -> str | None:
+    """Fallback: GET the creator page and scrape the campaign id from the page
+    bootstrap (gallery-dl's method). Tries both the bare and `/c/` vanity paths
+    Patreon redirects between. Never raises."""
+    jar = _load_cookie_jar(cookies_path)
+    headers = {"User-Agent": _USER_AGENT, "Accept": "text/html"}
+    # Try the bare vanity and every known creator-path prefix Patreon
+    # redirects between (c/, cw/) — the one the source used isn't known here
+    # (extract_vanity already stripped it).
+    page_urls = [f"https://www.patreon.com/{vanity}"]
+    page_urls += [f"https://www.patreon.com/{p}{vanity}" for p in _VANITY_PREFIXES]
+    for page_url in page_urls:
+        try:
+            resp = requests.get(
+                page_url,
+                headers=headers,
+                cookies=jar,
+                timeout=_TIMEOUT_SECONDS,
+                allow_redirects=True,
+            )
+        except requests.RequestException as exc:
+            log.warning("Patreon creator-page fetch failed for %s: %s", page_url, exc)
+            continue
+        if resp.status_code != 200:
+            continue
+        campaign_id = _scrape_campaign_id(resp.text)
+        if campaign_id:
+            log.info(
+                "Resolved Patreon vanity=%s → campaign_id=%s via creator page",
+                vanity, campaign_id,
+            )
+            return campaign_id
+    return None
+
+
+def _lookup_campaign_from_post(post_id: str, cookies_path: str | None) -> str | None:
+    """Resolve the owning campaign id from a single post id via the Patreon
+    post API (`/api/posts/<id>?include=campaign`). A `/posts/` source URL points
+    at one post, but a subscription walks the whole creator — so we follow the
+    post to its campaign. Never raises."""
+    jar = _load_cookie_jar(cookies_path)
+    headers = {"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"}
+    params = {"include": "campaign", "fields[campaign]": "name"}
+    url = f"{_POSTS_API}/{post_id}"
+    try:
+        resp = requests.get(
+            url, params=params, headers=headers, cookies=jar,
+            timeout=_TIMEOUT_SECONDS,
+        )
+    except requests.RequestException as exc:
+        log.warning("Patreon post lookup failed for post=%s: %s", post_id, exc)
+        return None
+    if resp.status_code != 200:
+        log.warning("Patreon post API returned HTTP %d for post=%s", resp.status_code, post_id)
+        return None
+    try:
+        payload = resp.json()
+    except ValueError:
+        return None
+    data = payload.get("data") if isinstance(payload, dict) else None
+    if not isinstance(data, dict):
+        return None
+    campaign = ((data.get("relationships") or {}).get("campaign") or {}).get("data") or {}
+    campaign_id = campaign.get("id")
+    if isinstance(campaign_id, str) and campaign_id:
+        log.info("Resolved Patreon post=%s → campaign_id=%s", post_id, campaign_id)
+        return campaign_id
+    return None
+
+
 async def resolve_campaign_id(
     vanity: str,
     cookies_path: str | None,
@@ -100,3 +228,55 @@ async def resolve_campaign_id(
     Never raises."""
     loop = asyncio.get_running_loop()
     return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
+
+
+async def resolve_campaign_from_post(
+    post_id: str, cookies_path: str | None,
+) -> str | None:
+    """Async wrapper around _lookup_campaign_from_post. Never raises."""
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(
+        None, _lookup_campaign_from_post, post_id, cookies_path
+    )
+
+
+def extract_vanity(url: str) -> str | None:
+    """The vanity slug from a Patreon creator URL, or None for an `id:` URL."""
+    m = _VANITY_RE.match(url or "")
+    return m.group(1) if m else None
+
+
+async def resolve_campaign_id_for_source(
+    url: str,
+    cookies_path: str | None,
+    overrides: dict | None,
+) -> tuple[str | None, str | None]:
+    """Resolve a Patreon source to its campaign id — the single resolution path
+    shared by the download ingester and the credential-verify probe.
+
+    Order: cached `patreon_campaign_id` override → an `id:<digits>` URL → a
+    `/posts/<id>` permalink (resolve the owning campaign from the post) → a
+    vanity lookup against the campaigns API. Returns
+    `(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None whenever
+    a lookup actually ran, so the caller caches it on the source (the
+    override/id: paths needed no lookup). `(None, None)` when unresolvable.
+    Never raises.
+    """
+    overrides = overrides or {}
+    cached = overrides.get("patreon_campaign_id")
+    if cached:
+        return cached, None
+    id_match = _ID_URL_RE.search(url or "")
+    if id_match:
+        return id_match.group(1), None
+    # A single-post URL → follow the post to its creator's campaign so the
+    # source subscribes to the whole feed (a subscription isn't one post).
+    post_match = _POST_URL_RE.search(url or "")
+    if post_match:
+        resolved = await resolve_campaign_from_post(post_match.group(1), cookies_path)
+        return resolved, resolved
+    vanity = extract_vanity(url)
+    if vanity:
+        resolved = await resolve_campaign_id(vanity, cookies_path)
+        return resolved, resolved
+    return None, None

backend/app/services/platform_lock.py

@@ -0,0 +1,60 @@
+"""Per-platform download concurrency cap.
+
+Some platforms (Patreon) are API-rate-sensitive enough that two *simultaneous*
+walks can trip the server's rate limit even with each source pacing its own
+requests. The platform-cooldown handles the AFTERMATH of a 429; this is the
+preventive half — it serializes downloads PER PLATFORM to one at a time.
+
+Different platforms still run concurrently up to the worker's concurrency; only
+a second walk on the SAME serialized platform waits. The lock lives in Redis
+(the Celery broker) with a TTL, so a SIGKILL'd worker can't wedge a platform —
+the lock auto-expires shortly after the download hard time limit.
+"""
+from __future__ import annotations
+
+import logging
+
+import redis
+
+from ..config import get_config
+
+log = logging.getLogger(__name__)
+
+# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
+# here: each runs as a self-pacing subprocess and they're lower-volume. The
+# native-ingester platforms are serialized (one paced scrape/API walk at a time).
+# Add a platform here to cap it to a single concurrent walk.
+SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
+
+_LOCK_PREFIX = "fc:download_lock:"
+
+_client: redis.Redis | None = None
+
+
+def _redis() -> redis.Redis:
+    # One client per worker process (Celery prefork forks before tasks run, so
+    # each process lazily builds its own). redis-py pools connections.
+    global _client
+    if _client is None:
+        _client = redis.from_url(get_config().celery_broker_url)
+    return _client
+
+
+def platform_lock(platform: str, *, ttl_seconds: int):
+    """A non-blocking Redis lock for `platform`, or None when the platform is
+    not serialized. Caller does `.acquire(blocking=False)` / `.release()`.
+
+    Returns None (rather than raising) on any Redis error so a broker hiccup
+    degrades to the prior behaviour (uncapped) instead of stalling downloads.
+    """
+    if platform not in SERIALIZED_PLATFORMS:
+        return None
+    try:
+        return _redis().lock(
+            f"{_LOCK_PREFIX}{platform}",
+            timeout=ttl_seconds,
+            blocking=False,
+        )
+    except redis.RedisError as exc:  # pragma: no cover - broker outage
+        log.warning("platform_lock unavailable for %s: %s", platform, exc)
+        return None

backend/app/services/post_feed_service.py

@@ -1,45 +1,44 @@
 """FC-3e: cursor-paginated read service for the Posts stream.
 
-Mirrors GalleryService.scroll's cursor encoding so the frontend pattern
-is identical: base64 of "<iso8601_sort_key>|<post_id>". Sort key is
-COALESCE(Post.post_date, Post.downloaded_at) so posts without a
-publish date sort by when we captured them.
+Uses the shared `pagination` cursor (base64 of "<iso8601_sort_key>|<id>") so
+every feed paginates identically. Sort key here is COALESCE(Post.post_date,
+Post.downloaded_at) so posts without a publish date sort by when we captured them.
 
 Pure read-surface; no writes. The service composes the post dict
-(including thumbnails from ImageRecord.primary_post_id and non-media
+(thumbnails from every image linked to the post — its own primary images
+plus cross-posted duplicates via image_provenance — and non-media
 attachments from PostAttachment) so the API layer can jsonify directly.
 """
 from __future__ import annotations
 
-import base64
-from datetime import datetime
+from html import unescape
 
 from sqlalchemy import and_, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ..models import Artist, ImageRecord, Post, PostAttachment, Source
+from ..models import (
+    Artist,
+    ExternalLink,
+    ImageProvenance,
+    ImageRecord,
+    Post,
+    PostAttachment,
+    Source,
+)
+from ..utils.html_sanitize import (
+    extract_img_srcs,
+    rewrite_img_srcs,
+    sanitize_post_html,
+)
+from ..utils.paths import filehash_from_url
 from ..utils.text import html_to_plain, truncate_at_word
-from .gallery_service import thumbnail_url
+from .gallery_service import image_url, thumbnail_url
+from .pagination import decode_cursor, encode_cursor
 
-CURSOR_SEPARATOR = "|"
 DESCRIPTION_LIMIT = 280
 THUMBNAIL_LIMIT = 6
 
 
-def encode_cursor(sort_key: datetime, post_id: int) -> str:
-    raw = f"{sort_key.isoformat()}{CURSOR_SEPARATOR}{post_id}"
-    return base64.urlsafe_b64encode(raw.encode()).decode()
-
-
-def decode_cursor(cursor: str) -> tuple[datetime, int]:
-    try:
-        raw = base64.urlsafe_b64decode(cursor.encode()).decode()
-        ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
-        return datetime.fromisoformat(ts_part), int(id_part)
-    except Exception as exc:
-        raise ValueError(f"invalid cursor: {cursor!r}") from exc
-
-
 def _sort_key():
     """Postgres COALESCE expression used in ORDER BY and WHERE clauses."""
     return func.coalesce(Post.post_date, Post.downloaded_at)
@@ -55,6 +54,7 @@ class PostFeedService:
         cursor: str | None = None,
         artist_id: int | None = None,
         platform: str | None = None,
+        q: str | None = None,
         limit: int = 24,
         direction: str = "older",
     ) -> dict:
@@ -62,7 +62,12 @@ class PostFeedService:
         time (default, infinite-scroll down); direction='newer' walks forward
         (scroll up in an anchored view). Items are always returned in feed
         (descending) order; `next_cursor` points to the far edge in the
-        requested direction (null when exhausted)."""
+        requested direction (null when exhausted).
+
+        `q` is a free-text filter (ILIKE substring over post_title OR
+        description) applied INSIDE the artist/platform scope, so a search
+        from the Browse bar stays within whatever artist is filtered in
+        view (operator-asked 2026-06-11)."""
         if limit < 1 or limit > 100:
             raise ValueError("limit must be between 1 and 100")
         if direction not in ("older", "newer"):
@@ -84,6 +89,12 @@ class PostFeedService:
             stmt = stmt.where(Post.artist_id == artist_id)
         if platform is not None:
             stmt = stmt.where(Source.platform == platform)
+        if q:
+            like = f"%{q}%"
+            stmt = stmt.where(or_(
+                Post.post_title.ilike(like),
+                Post.description.ilike(like),
+            ))
         if cursor:
             cur_ts, cur_id = decode_cursor(cursor)
             if direction == "older":
@@ -134,6 +145,7 @@ class PostFeedService:
         post_id: int,
         artist_id: int | None = None,
         platform: str | None = None,
+        q: str | None = None,
         limit: int = 12,
     ) -> dict | None:
         """A window centered on `post_id`: up to `limit` newer posts + the
@@ -153,11 +165,11 @@ class PostFeedService:
 
         older = await self.scroll(
             cursor=anchor_cursor, artist_id=artist_id, platform=platform,
-            limit=limit, direction="older",
+            q=q, limit=limit, direction="older",
         )
         newer = await self.scroll(
             cursor=anchor_cursor, artist_id=artist_id, platform=platform,
-            limit=limit, direction="newer",
+            q=q, limit=limit, direction="newer",
         )
         thumbs_map = await self._thumbnails_for([anchor_post.id])
         atts_map = await self._attachments_for([anchor_post.id])
@@ -188,8 +200,71 @@ class PostFeedService:
         atts_map = await self._attachments_for([post.id])
         item = self._to_dict(post, artist, source, thumbs_map, atts_map)
         item["description_full"] = html_to_plain(post.description)
+        # Sanitized HTML body for faithful (semantic) rendering in the post view;
+        # detail-only (the feed list stays lightweight plain text). None when the
+        # post has no body. Inline `<img>` sources are remapped to locally-served
+        # copies (#830 Phase 2) so the body never hotlinks the public CDN.
+        item["description_html"] = await self._localize_inline_images(
+            sanitize_post_html(post.description), post.artist_id,
+        )
+        item["external_links"] = await self._external_links_for(post.id)
         return item
 
+    async def _localize_inline_images(
+        self, html: str | None, artist_id: int | None,
+    ) -> str | None:
+        """Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
+
+        The join key is the CDN filehash the downloader persisted on each
+        ImageRecord (source_filehash): for every body image whose filehash maps
+        to a stored image of THIS artist, swap the src to /images/<path>. Images
+        we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
+        they keep hotlinking, which is the prior behavior. Scoped to the post's
+        artist so one creator's body never resolves to another's file."""
+        if not html or artist_id is None:
+            return html
+        srcs = extract_img_srcs(html)
+        if not srcs:
+            return html
+        # filehash -> the raw (as-in-HTML) src strings carrying it. A body can
+        # repeat the same image; keep every raw form so each is substituted.
+        by_hash: dict[str, list[str]] = {}
+        for raw in srcs:
+            fh = filehash_from_url(unescape(raw))
+            if fh:
+                by_hash.setdefault(fh, []).append(raw)
+        if not by_hash:
+            return html
+        rows = (await self.session.execute(
+            select(ImageRecord.source_filehash, ImageRecord.path)
+            .where(
+                ImageRecord.artist_id == artist_id,
+                ImageRecord.source_filehash.in_(list(by_hash)),
+            )
+        )).all()
+        replace: dict[str, str] = {}
+        for fh, path in rows:
+            for raw in by_hash.get(fh, ()):
+                replace[raw] = image_url(path)
+        return rewrite_img_srcs(html, replace)
+
+    async def _external_links_for(self, post_id: int) -> list[dict]:
+        """Off-platform file-host links recorded for a post (detail-only). Each
+        carries its host, full url, label, and download status so the post view
+        can surface them (and, later, a retry/download affordance)."""
+        rows = (await self.session.execute(
+            select(ExternalLink)
+            .where(ExternalLink.post_id == post_id)
+            .order_by(ExternalLink.id.asc())
+        )).scalars().all()
+        return [
+            {
+                "id": e.id, "host": e.host, "url": e.url,
+                "label": e.label, "status": e.status,
+            }
+            for e in rows
+        ]
+
     # --- composition helpers ---------------------------------------------
 
     async def _thumbnails_for(
@@ -205,28 +280,50 @@ class PostFeedService:
         """
         if not post_ids:
             return {}
+        # A post shows EVERY image linked to it — both its own primary images and
+        # cross-posted duplicates linked via image_provenance (a near-dup of an
+        # existing image gets a provenance row for the new post, not dropped; see
+        # image_provenance docstring + the importer enrich-on-duplicate path). The
+        # UNION dedups (image, post) pairs and also keeps any legacy image that has
+        # a primary_post_id but no provenance row. Partition the window on the
+        # link's post_id, not ImageRecord.primary_post_id, so a duplicate counts
+        # under each post it belongs to.
+        links = (
+            select(
+                ImageProvenance.image_record_id.label("image_id"),
+                ImageProvenance.post_id.label("post_id"),
+            )
+            .where(ImageProvenance.post_id.in_(post_ids))
+            .union(
+                select(
+                    ImageRecord.id.label("image_id"),
+                    ImageRecord.primary_post_id.label("post_id"),
+                ).where(ImageRecord.primary_post_id.in_(post_ids))
+            )
+            .subquery()
+        )
         # Rank images within each post; cap at `limit` rows per post when
         # limit is set, return all when limit is None.
         ranked = (
             select(
                 ImageRecord.id,
-                ImageRecord.primary_post_id,
+                links.c.post_id,
                 ImageRecord.sha256,
                 ImageRecord.mime,
                 ImageRecord.thumbnail_path,
                 func.row_number().over(
-                    partition_by=ImageRecord.primary_post_id,
+                    partition_by=links.c.post_id,
                     order_by=ImageRecord.id.asc(),
                 ).label("rn"),
                 func.count(ImageRecord.id).over(
-                    partition_by=ImageRecord.primary_post_id,
+                    partition_by=links.c.post_id,
                 ).label("total"),
             )
-            .where(ImageRecord.primary_post_id.in_(post_ids))
+            .join(ImageRecord, ImageRecord.id == links.c.image_id)
             .subquery()
         )
         stmt = select(
-            ranked.c.id, ranked.c.primary_post_id,
+            ranked.c.id, ranked.c.post_id,
             ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total,
         )
         if limit is not None:

backend/app/services/provenance_service.py

@@ -66,6 +66,10 @@ class ProvenanceService:
         ).scalars().all()
         return [_attachment_dict(a) for a in rows]
 
+    async def _attachment_by_id(self, attachment_id: int) -> list[dict]:
+        att = await self.session.get(PostAttachment, attachment_id)
+        return [_attachment_dict(att)] if att is not None else []
+
     async def for_image(self, image_id: int) -> dict | None:
         rec = await self.session.get(ImageRecord, image_id)
         if rec is None:
@@ -85,7 +89,33 @@ class ProvenanceService:
         )
         rows = (await self.session.execute(stmt)).all()
         post_ids = [ip.post_id for ip, _p, _s, _a in rows]
-        attachments = await self._attachments_for_posts(post_ids)
+        # Prefer the EXACT archive this file came out of (milestone #87): if the
+        # originating post's provenance row records from_attachment_id, the image
+        # was extracted from that one .zip/.rar, so show only it — not the dozens
+        # of unrelated archives a "High Resolution Files" bundle post carries.
+        from_att_id = next(
+            (
+                ip.from_attachment_id
+                for ip, _p, _s, _a in rows
+                if ip.post_id == rec.primary_post_id
+                and ip.from_attachment_id is not None
+            ),
+            None,
+        )
+        if from_att_id is not None:
+            attachments = await self._attachment_by_id(from_att_id)
+        else:
+            # No recorded containing archive (loose download, or pre-backfill):
+            # scope to the originating post only, not every pHash-linked post.
+            # primary_post_id is the post this file was actually captured from;
+            # fall back to all linked posts when it's unset (older rows /
+            # filesystem imports).
+            attach_post_ids = (
+                [rec.primary_post_id]
+                if rec.primary_post_id is not None
+                else post_ids
+            )
+            attachments = await self._attachments_for_posts(attach_post_ids)
         return {
             "image_id": image_id,
             "provenance": [

backend/app/services/series_match_service.py

@@ -0,0 +1,338 @@
+"""FC-6.3 — assisted continuation matcher.
+
+Scores a (post, candidate series) pair from several weighted signals; above the
+configured threshold it records a *pending* SeriesSuggestion. Confirm-only — the
+operator accepts (post becomes a chapter) or dismisses. No single signal gates;
+the score is an additive weighted sum, each signal a 0..1 strength.
+
+The learned "title pattern" isn't persisted — it's derived on the fly from the
+post titles already in a series, so it sharpens automatically as more posts are
+confirmed into the series.
+"""
+
+import re
+import time
+
+from sqlalchemy import and_, func, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..models import ImageRecord, Post, Tag, TagKind
+from ..models.series_page import SeriesPage
+from ..models.series_suggestion import SeriesSuggestion
+from ..models.tag import image_tag
+from .page_number_parser import parse_page_range
+from .series_service import SeriesError, SeriesService
+
+# Additive signal weights (sum to 1.0 → max score 1.0). Kept as constants;
+# only the on/off + threshold are operator-tunable (sensitivity is the knob
+# that matters; per-signal weights are an over-tune for v1).
+WEIGHTS = {"title": 0.40, "artist": 0.20, "pages": 0.25, "tags": 0.15}
+
+_DISTINCTIVE_KINDS = (TagKind.character, TagKind.series, TagKind.fandom)
+_MAX_CANDIDATES = 50
+
+# Strip installment markers so titles collapse to their stable stem.
+_PAGE_TOKEN = re.compile(r"\b(?:pages?|pgs?|pp\.?)\s*\d+(?:\s*[-–—/]\s*\d+)?", re.I)
+_BRACKET_NUM = re.compile(r"[\[(]\s*\d+\s*(?:/\s*\d+)?\s*[\])]")
+_TRAILING_NUM = re.compile(r"[\s\-_#]*\d+\s*$")
+
+
+def normalize_title(title: str | None) -> str:
+    if not title:
+        return ""
+    t = _PAGE_TOKEN.sub("", title)
+    t = _BRACKET_NUM.sub("", t)
+    t = _TRAILING_NUM.sub("", t)
+    return " ".join(t.split()).strip().lower()
+
+
+def _common_prefix(strings: list[str]) -> str:
+    strings = [s for s in strings if s]
+    if not strings:
+        return ""
+    pre = strings[0]
+    for s in strings[1:]:
+        i = 0
+        while i < len(pre) and i < len(s) and pre[i] == s[i]:
+            i += 1
+        pre = pre[:i]
+        if not pre:
+            break
+    return pre.strip()
+
+
+def title_signal(series_titles: list[str], post_title: str | None) -> float:
+    """Overlap of the post title against the series' title stem (the longest
+    common prefix of its known titles, page/installment markers removed)."""
+    norm = [normalize_title(t) for t in series_titles]
+    norm = [t for t in norm if t]
+    pt = normalize_title(post_title)
+    if not norm or not pt:
+        return 0.0
+    stem = _common_prefix(norm)
+    if len(stem) < 3:
+        stem = max(norm, key=len)
+    i = 0
+    while i < len(stem) and i < len(pt) and stem[i] == pt[i]:
+        i += 1
+    return min(1.0, i / max(len(stem), 4))
+
+
+def pages_signal(series_max_stated_end: int | None, post_start: int | None) -> float:
+    """1.0 when the post's first page continues right after the series' last
+    stated page; partial for a near-continuation; 0 otherwise."""
+    if series_max_stated_end is None or post_start is None:
+        return 0.0
+    diff = post_start - series_max_stated_end
+    if diff == 1:
+        return 1.0
+    if 2 <= diff <= 3:
+        return 0.5
+    return 0.0
+
+
+def tags_signal(shared_distinctive: int) -> float:
+    if shared_distinctive <= 0:
+        return 0.0
+    return min(1.0, shared_distinctive / 3.0)
+
+
+def weighted_score(signals: dict) -> float:
+    return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4)
+
+
+class SeriesMatchService:
+    def __init__(self, session: AsyncSession):
+        self.session = session
+
+    async def _post_image_ids(self, post_id: int) -> list[int]:
+        rows = (
+            await self.session.execute(
+                select(ImageRecord.id).where(ImageRecord.primary_post_id == post_id)
+            )
+        ).scalars().all()
+        return list(rows)
+
+    async def _series_image_ids(self, series_tag_id: int) -> set[int]:
+        rows = (
+            await self.session.execute(
+                select(SeriesPage.image_id).where(
+                    SeriesPage.series_tag_id == series_tag_id
+                )
+            )
+        ).scalars().all()
+        return set(rows)
+
+    async def _series_post_titles(self, series_tag_id: int) -> list[str]:
+        rows = (
+            await self.session.execute(
+                select(Post.post_title)
+                .select_from(SeriesPage)
+                .join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
+                .join(Post, Post.id == ImageRecord.primary_post_id)
+                .where(
+                    and_(
+                        SeriesPage.series_tag_id == series_tag_id,
+                        Post.post_title.isnot(None),
+                    )
+                )
+                .distinct()
+            )
+        ).scalars().all()
+        return [t for t in rows if t]
+
+    async def _series_max_stated_end(self, series_tag_id: int) -> int | None:
+        # Highest printed page across the series' pages (FC-6.x: stated pages
+        # live on series_page now; chapters are cosmetic dividers).
+        return await self.session.scalar(
+            select(func.max(SeriesPage.stated_page)).where(
+                SeriesPage.series_tag_id == series_tag_id
+            )
+        )
+
+    async def _distinctive_tags(self, image_ids: list[int] | set[int]) -> set[int]:
+        ids = list(image_ids)
+        if not ids:
+            return set()
+        rows = (
+            await self.session.execute(
+                select(image_tag.c.tag_id)
+                .join(Tag, Tag.id == image_tag.c.tag_id)
+                .where(
+                    and_(
+                        image_tag.c.image_record_id.in_(ids),
+                        Tag.kind.in_(_DISTINCTIVE_KINDS),
+                    )
+                )
+                .distinct()
+            )
+        ).scalars().all()
+        return set(rows)
+
+    async def _candidate_series(self, artist_id: int) -> list[int]:
+        """Series that already contain a page by this artist — cross-artist
+        series are rare, so same-artist is the candidate bound."""
+        rows = (
+            await self.session.execute(
+                select(SeriesPage.series_tag_id)
+                .join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
+                .where(ImageRecord.artist_id == artist_id)
+                .distinct()
+            )
+        ).scalars().all()
+        return list(rows)
+
+    async def _decided_series(self, post_id: int) -> set[int]:
+        rows = (
+            await self.session.execute(
+                select(SeriesSuggestion.series_tag_id).where(
+                    and_(
+                        SeriesSuggestion.post_id == post_id,
+                        SeriesSuggestion.status.in_(["added", "dismissed"]),
+                    )
+                )
+            )
+        ).scalars().all()
+        return set(rows)
+
+    async def match_post(self, post_id: int, *, threshold: float) -> int:
+        """Score a post against its artist's series; upsert pending suggestions
+        for those at/above threshold. Returns the number written."""
+        post = await self.session.get(Post, post_id)
+        if post is None or post.artist_id is None:
+            return 0
+        post_images = await self._post_image_ids(post_id)
+        if not post_images:
+            return 0
+        post_image_set = set(post_images)
+        rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}")
+        post_start = rng[0] if rng else None
+        post_dtags = await self._distinctive_tags(post_images)
+
+        decided = await self._decided_series(post_id)
+        candidates = await self._candidate_series(post.artist_id)
+        made = 0
+        for sid in candidates[:_MAX_CANDIDATES]:
+            if sid in decided:
+                continue
+            s_images = await self._series_image_ids(sid)
+            if post_image_set & s_images:
+                continue  # the post is already (partly) in this series
+            signals = {
+                "title": title_signal(
+                    await self._series_post_titles(sid), post.post_title
+                ),
+                "artist": 1.0,  # candidates are same-artist by construction
+                "pages": pages_signal(
+                    await self._series_max_stated_end(sid), post_start
+                ),
+                "tags": tags_signal(
+                    len(post_dtags & await self._distinctive_tags(s_images))
+                ),
+            }
+            score = weighted_score(signals)
+            if score < threshold:
+                continue
+            await self.session.execute(
+                pg_insert(SeriesSuggestion)
+                .values(
+                    post_id=post_id, series_tag_id=sid,
+                    score=score, signals=signals, status="pending",
+                )
+                .on_conflict_do_update(
+                    constraint="uq_series_suggestion_post_series",
+                    set_={"score": score, "signals": signals, "status": "pending"},
+                    where=SeriesSuggestion.status == "pending",
+                )
+            )
+            made += 1
+        return made
+
+    # ---- queue ops --------------------------------------------------------
+
+    async def list_pending(self) -> list[dict]:
+        rows = (
+            await self.session.execute(
+                select(
+                    SeriesSuggestion.id,
+                    SeriesSuggestion.post_id,
+                    SeriesSuggestion.series_tag_id,
+                    SeriesSuggestion.score,
+                    SeriesSuggestion.signals,
+                    Post.post_title,
+                    Post.external_post_id,
+                    Tag.name.label("series_name"),
+                )
+                .join(Post, Post.id == SeriesSuggestion.post_id)
+                .join(Tag, Tag.id == SeriesSuggestion.series_tag_id)
+                .where(SeriesSuggestion.status == "pending")
+                .order_by(SeriesSuggestion.score.desc())
+            )
+        ).all()
+        return [
+            {
+                "id": r.id,
+                "post_id": r.post_id,
+                "series_tag_id": r.series_tag_id,
+                "series_name": r.series_name,
+                "post_title": r.post_title or f"Post {r.external_post_id}",
+                "score": r.score,
+                "signals": r.signals or {},
+            }
+            for r in rows
+        ]
+
+    async def accept(self, suggestion_id: int) -> dict:
+        s = await self.session.get(SeriesSuggestion, suggestion_id)
+        if s is None:
+            raise SeriesError(f"suggestion {suggestion_id} not found")
+        if s.status != "pending":
+            raise SeriesError(f"suggestion {suggestion_id} is already {s.status}")
+        out = await SeriesService(self.session).add_post(
+            s.series_tag_id, s.post_id
+        )
+        s.status = "added"
+        self.session.add(s)
+        return out
+
+    async def dismiss(self, suggestion_id: int) -> None:
+        s = await self.session.get(SeriesSuggestion, suggestion_id)
+        if s is None:
+            raise SeriesError(f"suggestion {suggestion_id} not found")
+        if s.status == "pending":
+            s.status = "dismissed"
+            self.session.add(s)
+
+    async def rescan(
+        self, *, threshold: float, time_budget_seconds: float | None = None,
+        after_post_id: int = 0,
+    ) -> dict:
+        """Score every post (id > after_post_id) against its artist's series,
+        time-boxed + resumable like the other long maintenance sweeps."""
+        post_ids = (
+            await self.session.execute(
+                select(Post.id)
+                .where(Post.id > after_post_id)
+                .order_by(Post.id.asc())
+            )
+        ).scalars().all()
+        summary = {
+            "scanned": 0, "suggested": 0,
+            "partial": False, "resume_after_id": after_post_id,
+        }
+        start = time.monotonic()
+        for pid in post_ids:
+            summary["scanned"] += 1
+            summary["resume_after_id"] = pid
+            summary["suggested"] += await self.match_post(pid, threshold=threshold)
+            await self.session.commit()  # commit per post so progress survives
+            if (
+                time_budget_seconds is not None
+                and time.monotonic() - start >= time_budget_seconds
+            ):
+                summary["partial"] = True
+                break
+        else:
+            summary["partial"] = False
+        return summary

backend/app/services/series_service.py

@@ -1,15 +1,26 @@
-"""Series = a series-kind Tag + ordered series_page membership.
+"""Series = a series-kind Tag + ONE flat, series-global ordered run of pages,
+with optional cosmetic chapter DIVIDERS over it (FC-6.x reframe, #789).
+
+Reading order is series_page.page_number alone (rewritten 1..N wholesale on any
+reorder; page_number is not unique, so a reorder never transiently collides). A
+chapter is NOT a container — it is a labeled divider anchored to the page that
+begins it (series_chapter.anchor_page_id); a page's chapter is derived as the
+nearest preceding divider. An image lives in at most one series
+(series_page.image_id is UNIQUE).
 
 All mutations are Core/set-based and run in the request transaction.
 """
 
 from sqlalchemy import and_, func, select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ..models import ImageRecord, Tag, TagKind
+from ..models import Artist, ImageRecord, Post, Tag, TagKind
+from ..models.series_chapter import SeriesChapter
 from ..models.series_page import SeriesPage
-from .gallery_service import thumbnail_url
+from .gallery_service import image_url, thumbnail_url
+from .page_number_parser import parse_page_range
 
 
 class SeriesError(ValueError):
@@ -21,6 +32,8 @@ class SeriesService:
     def __init__(self, session: AsyncSession):
         self.session = session
 
+    # ---- guards / helpers -------------------------------------------------
+
     async def _require_series(self, series_tag_id: int) -> Tag:
         tag = await self.session.get(Tag, series_tag_id)
         if tag is None:
@@ -32,7 +45,7 @@ class SeriesService:
     @staticmethod
     def _clean_ids(ids: list[int]) -> list[int]:
         if not ids:
-            raise SeriesError("image_ids must be a non-empty list")
+            raise SeriesError("ids must be a non-empty list")
         if len(ids) > 500:
             raise SeriesError("selection too large (max 500)")
         seen: set[int] = set()
@@ -43,16 +56,106 @@ class SeriesService:
                 out.append(int(x))
         return out
 
-    async def _member_order(self, series_tag_id: int) -> list[int]:
+    async def _page_order(self, series_tag_id: int) -> list[int]:
+        """The series' PLACED image_ids in reading order (series-global
+        page_number). Pending (staged) pages are excluded — they have no order
+        yet."""
         rows = (
             await self.session.execute(
                 select(SeriesPage.image_id)
-                .where(SeriesPage.series_tag_id == series_tag_id)
-                .order_by(SeriesPage.page_number.asc())
+                .where(
+                    and_(
+                        SeriesPage.series_tag_id == series_tag_id,
+                        SeriesPage.status == "placed",
+                    )
+                )
+                .order_by(
+                    SeriesPage.page_number.asc().nulls_last(),
+                    SeriesPage.id.asc(),
+                )
             )
         ).scalars().all()
         return list(rows)
 
+    async def _page_id_for(self, series_tag_id: int, image_id: int) -> int:
+        """series_page.id for an image in the series (raises if not a member)."""
+        pid = await self.session.scalar(
+            select(SeriesPage.id).where(
+                and_(
+                    SeriesPage.series_tag_id == series_tag_id,
+                    SeriesPage.image_id == image_id,
+                )
+            )
+        )
+        if pid is None:
+            raise SeriesError(f"image {image_id} is not in this series")
+        return pid
+
+    async def _require_divider(self, series_tag_id: int, divider_id: int):
+        row = (
+            await self.session.execute(
+                select(SeriesChapter).where(
+                    and_(
+                        SeriesChapter.id == divider_id,
+                        SeriesChapter.series_tag_id == series_tag_id,
+                    )
+                )
+            )
+        ).scalar_one_or_none()
+        if row is None:
+            raise SeriesError(
+                f"chapter {divider_id} is not in series {series_tag_id}"
+            )
+        return row
+
+    # ---- read -------------------------------------------------------------
+
+    @staticmethod
+    def _part_gaps(dividers: list[dict]) -> list[dict]:
+        """Missing-Part hints between consecutive dividers whose stated_part
+        numbers jump by more than 1 (e.g. Part 1 then Part 3). Only dividers
+        that carry a stated_part participate."""
+        out: list[dict] = []
+        prev = None
+        for d in dividers:
+            cur = d["stated_part"]
+            if cur is None:
+                continue
+            if prev is not None and cur > prev["stated_part"] + 1:
+                out.append(
+                    {
+                        "after_divider_id": prev["id"],
+                        "start": prev["stated_part"] + 1,
+                        "end": cur - 1,
+                    }
+                )
+            prev = d
+        return out
+
+    @staticmethod
+    def _page_gaps(pages: list[dict]) -> list[dict]:
+        """Missing-number gaps between consecutive NUMBERED placed pages — one
+        entry per gap (e.g. pages 2 then 5 → one gap 3–4). The UI renders each as
+        a single block whose edges accept a dropped page. Unnumbered pages (at
+        the tail) don't participate."""
+        out: list[dict] = []
+        prev = None
+        for p in pages:
+            n = p["page_number"]
+            if n is None:
+                continue
+            if prev is not None and n > prev["page_number"] + 1:
+                out.append(
+                    {
+                        "after_image_id": prev["image_id"],
+                        "before_image_id": p["image_id"],
+                        "from": prev["page_number"] + 1,
+                        "to": n - 1,
+                    }
+                )
+            prev = p
+        return out
+
     async def list_pages(self, series_tag_id: int) -> dict:
         tag = await self._require_series(series_tag_id)
         rows = (
@@ -60,32 +163,156 @@ class SeriesService:
                 select(
                     SeriesPage.image_id,
                     SeriesPage.page_number,
+                    SeriesPage.stated_page,
                     ImageRecord.sha256,
                     ImageRecord.mime,
                     ImageRecord.path,
                     ImageRecord.thumbnail_path,
+                    ImageRecord.primary_post_id,
+                    Post.post_title,
                 )
+                .select_from(SeriesPage)
                 .join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
-                .where(SeriesPage.series_tag_id == series_tag_id)
+                .outerjoin(Post, Post.id == ImageRecord.primary_post_id)
+                .where(
+                    and_(
+                        SeriesPage.series_tag_id == series_tag_id,
+                        SeriesPage.status == "placed",
+                    )
+                )
+                .order_by(
+                    SeriesPage.page_number.asc().nulls_last(),
+                    SeriesPage.id.asc(),
+                )
+            )
+        ).all()
+        pages = [
+            {
+                "image_id": r.image_id,
+                "page_number": r.page_number,
+                "stated_page": r.stated_page,
+                "thumbnail_url": thumbnail_url(
+                    r.thumbnail_path, r.sha256, r.mime
+                ),
+                "image_url": image_url(r.path),
+                "source_post": (
+                    {"id": r.primary_post_id, "title": r.post_title}
+                    if r.primary_post_id is not None
+                    else None
+                ),
+            }
+            for r in rows
+        ]
+
+        # Dividers, ordered by the global position of their anchor page.
+        drows = (
+            await self.session.execute(
+                select(
+                    SeriesChapter.id,
+                    SeriesChapter.title,
+                    SeriesChapter.stated_part,
+                    SeriesPage.image_id.label("anchor_image_id"),
+                    SeriesPage.page_number,
+                )
+                .join(SeriesPage, SeriesPage.id == SeriesChapter.anchor_page_id)
+                .where(SeriesChapter.series_tag_id == series_tag_id)
                 .order_by(SeriesPage.page_number.asc())
             )
         ).all()
-        return {
-            "series": {"id": tag.id, "name": tag.name},
-            "pages": [
+        dividers = [
+            {
+                "id": r.id,
+                "anchor_image_id": r.anchor_image_id,
+                "page_number": r.page_number,
+                "title": r.title,
+                "stated_part": r.stated_part,
+            }
+            for r in drows
+        ]
+
+        # Pending (staged-from-post) pages, grouped by their source post so the
+        # operator can drop junk and place the keepers into the run.
+        prows = (
+            await self.session.execute(
+                select(
+                    SeriesPage.image_id,
+                    SeriesPage.stated_page,
+                    ImageRecord.sha256,
+                    ImageRecord.mime,
+                    ImageRecord.path,
+                    ImageRecord.thumbnail_path,
+                    ImageRecord.primary_post_id,
+                    Post.post_title,
+                )
+                .select_from(SeriesPage)
+                .join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
+                .outerjoin(Post, Post.id == ImageRecord.primary_post_id)
+                .where(
+                    and_(
+                        SeriesPage.series_tag_id == series_tag_id,
+                        SeriesPage.status == "pending",
+                    )
+                )
+                .order_by(
+                    ImageRecord.primary_post_id.asc().nulls_last(),
+                    SeriesPage.id.asc(),
+                )
+            )
+        ).all()
+        pending_groups: list[dict] = []
+        by_post: dict = {}
+        for r in prows:
+            grp = by_post.get(r.primary_post_id)
+            if grp is None:
+                grp = {
+                    "post": (
+                        {"id": r.primary_post_id, "title": r.post_title}
+                        if r.primary_post_id is not None
+                        else None
+                    ),
+                    "start_page": None,
+                    "pages": [],
+                }
+                by_post[r.primary_post_id] = grp
+                pending_groups.append(grp)
+            grp["pages"].append(
                 {
                     "image_id": r.image_id,
-                    "page_number": r.page_number,
-                    "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
-                    "image_url": f"/images/{r.path.split('/images/', 1)[-1]}",
+                    "stated_page": r.stated_page,
+                    "thumbnail_url": thumbnail_url(
+                        r.thumbnail_path, r.sha256, r.mime
+                    ),
+                    "image_url": image_url(r.path),
                 }
-                for r in rows
-            ],
+            )
+            # Group start page = the post's parsed starting page (stored on the
+            # staged pages); surfaced as the default for "Place from page N".
+            if r.stated_page is not None:
+                cur = grp["start_page"]
+                grp["start_page"] = (
+                    r.stated_page if cur is None else min(cur, r.stated_page)
+                )
+
+        return {
+            "series": {"id": tag.id, "name": tag.name},
+            "pages": pages,
+            "gaps": self._page_gaps(pages),
+            "dividers": dividers,
+            "pending": pending_groups,
+            "part_gaps": self._part_gaps(dividers),
         }
 
+    # ---- pages ------------------------------------------------------------
+
     async def add_images(
-        self, series_tag_id: int, image_ids: list[int]
+        self,
+        series_tag_id: int,
+        image_ids: list[int],
+        stated_pages: dict[int, int] | None = None,
     ) -> int:
+        """Append images as pages at the END of the series' flat run. Images
+        already in THIS series are left untouched; images in another series are
+        moved here (image_id is UNIQUE)."""
         await self._require_series(series_tag_id)
         ids = self._clean_ids(image_ids)
         existing = dict(
@@ -96,29 +323,25 @@ class SeriesService:
                 )
             ).all()
         )
-        # Already in THIS series → leave untouched (no churn).
         to_add = [i for i in ids if existing.get(i) != series_tag_id]
         if not to_add:
             return 0
-        # Remove any prior membership for the to_add images (the "move").
+        # Move: drop any prior membership for these images (UNIQUE image_id). A
+        # dropped page that anchored a divider in another series cascades it away.
         await self.session.execute(
-            SeriesPage.__table__.delete().where(
-                SeriesPage.image_id.in_(to_add)
-            )
+            SeriesPage.__table__.delete().where(SeriesPage.image_id.in_(to_add))
         )
-        max_pn = (
-            await self.session.scalar(
-                select(func.coalesce(func.max(SeriesPage.page_number), 0))
-                .where(SeriesPage.series_tag_id == series_tag_id)
-            )
-        ) or 0
+        max_pn = await self._max_placed_page_number(series_tag_id)
+        sp = stated_pages or {}
         await self.session.execute(
             pg_insert(SeriesPage).values(
                 [
                     {
                         "series_tag_id": series_tag_id,
                         "image_id": iid,
+                        "status": "placed",
                         "page_number": max_pn + offset,
+                        "stated_page": sp.get(iid),
                     }
                     for offset, iid in enumerate(to_add, start=1)
                 ]
@@ -126,6 +349,20 @@ class SeriesService:
         )
         return len(to_add)
 
+    async def _max_placed_page_number(self, series_tag_id: int) -> int:
+        return (
+            await self.session.scalar(
+                select(
+                    func.coalesce(func.max(SeriesPage.page_number), 0)
+                ).where(
+                    and_(
+                        SeriesPage.series_tag_id == series_tag_id,
+                        SeriesPage.status == "placed",
+                    )
+                )
+            )
+        ) or 0
+
     async def remove_images(
         self, series_tag_id: int, image_ids: list[int]
     ) -> int:
@@ -139,19 +376,274 @@ class SeriesService:
                 )
             )
         )
+        # Dividers anchored to a removed page cascade away (chapter merges into
+        # the preceding run). Page numbers are operator-set and sparse, so we do
+        # NOT renumber — removing a page just leaves a gap.
         return res.rowcount or 0
 
-    async def reorder(
-        self, series_tag_id: int, ordered_image_ids: list[int]
+    async def set_page_number(
+        self, series_tag_id: int, image_id: int, page_number: int | None
     ) -> None:
+        """Set one placed page's number — the operator's value: sparse, gaps
+        allowed, or None to leave it unnumbered (sorts to the end). No other page
+        is touched; reading order follows the numbers."""
         await self._require_series(series_tag_id)
-        ordered = self._clean_ids(ordered_image_ids)
-        current = set(await self._member_order(series_tag_id))
-        if set(ordered) != current or len(ordered) != len(current):
-            raise SeriesError(
-                "ordered image_ids must exactly match series membership"
+        res = await self.session.execute(
+            update(SeriesPage)
+            .where(
+                and_(
+                    SeriesPage.series_tag_id == series_tag_id,
+                    SeriesPage.image_id == image_id,
+                    SeriesPage.status == "placed",
+                )
             )
-        for idx, iid in enumerate(ordered, start=1):
+            .values(page_number=page_number)
+        )
+        if (res.rowcount or 0) == 0:
+            raise SeriesError(
+                f"image {image_id} is not a placed page of this series"
+            )
+
+    async def set_cover(self, series_tag_id: int, image_id: int) -> None:
+        """Cover = the lowest-numbered page. Renumber the chosen image to sit
+        just before the current minimum so it sorts first."""
+        await self._require_series(series_tag_id)
+        await self._page_id_for(series_tag_id, image_id)  # membership guard
+        min_other = await self.session.scalar(
+            select(func.min(SeriesPage.page_number)).where(
+                and_(
+                    SeriesPage.series_tag_id == series_tag_id,
+                    SeriesPage.status == "placed",
+                    SeriesPage.image_id != image_id,
+                )
+            )
+        )
+        new_pn = (min_other - 1) if min_other is not None else 1
+        await self.session.execute(
+            update(SeriesPage)
+            .where(
+                and_(
+                    SeriesPage.series_tag_id == series_tag_id,
+                    SeriesPage.image_id == image_id,
+                )
+            )
+            .values(page_number=new_pn)
+        )
+
+    # ---- chapter dividers -------------------------------------------------
+
+    async def create_divider(
+        self,
+        series_tag_id: int,
+        anchor_image_id: int,
+        *,
+        title: str | None = None,
+        stated_part: int | None = None,
+    ) -> dict:
+        """Place a chapter divider before the page for anchor_image_id ("a new
+        chapter starts here"). One divider per page (UNIQUE anchor)."""
+        await self._require_series(series_tag_id)
+        page_id = await self._page_id_for(series_tag_id, anchor_image_id)
+        try:
+            async with self.session.begin_nested():
+                res = await self.session.execute(
+                    pg_insert(SeriesChapter)
+                    .values(
+                        series_tag_id=series_tag_id,
+                        anchor_page_id=page_id,
+                        title=title,
+                        stated_part=stated_part,
+                    )
+                    .returning(SeriesChapter.id)
+                )
+                divider_id = res.scalar_one()
+        except IntegrityError as exc:
+            raise SeriesError(
+                "a chapter divider already starts at that page"
+            ) from exc
+        return {"id": divider_id, "anchor_image_id": anchor_image_id}
+
+    async def update_divider(
+        self,
+        series_tag_id: int,
+        divider_id: int,
+        *,
+        title: str | None = None,
+        stated_part: int | None = None,
+        anchor_image_id: int | None = None,
+        set_title: bool = False,
+        set_part: bool = False,
+        set_anchor: bool = False,
+    ) -> None:
+        """Partial divider edit. set_* flags say which fields to write (so None
+        can be written explicitly, e.g. clearing a stated_part). set_anchor moves
+        the divider to sit before a different page."""
+        await self._require_series(series_tag_id)
+        await self._require_divider(series_tag_id, divider_id)
+        values: dict = {}
+        if set_title:
+            values["title"] = title
+        if set_part:
+            values["stated_part"] = stated_part
+        if set_anchor:
+            if anchor_image_id is None:
+                raise SeriesError("anchor_image_id required to move a divider")
+            values["anchor_page_id"] = await self._page_id_for(
+                series_tag_id, anchor_image_id
+            )
+        if not values:
+            return
+        try:
+            async with self.session.begin_nested():
+                await self.session.execute(
+                    update(SeriesChapter)
+                    .where(SeriesChapter.id == divider_id)
+                    .values(**values)
+                )
+        except IntegrityError as exc:
+            raise SeriesError(
+                "a chapter divider already starts at that page"
+            ) from exc
+
+    async def delete_divider(
+        self, series_tag_id: int, divider_id: int
+    ) -> None:
+        """Remove a divider — the chapter merges into the run above it. Pages
+        are untouched (dividers own nothing)."""
+        await self._require_series(series_tag_id)
+        await self._require_divider(series_tag_id, divider_id)
+        await self.session.execute(
+            SeriesChapter.__table__.delete().where(
+                SeriesChapter.id == divider_id
+            )
+        )
+
+    # ---- post → series (FC-6.2) ------------------------------------------
+
+    async def _post_images_ordered(self, post_id: int) -> list[int]:
+        """A post's images in capture order (ImageRecord.primary_post_id), the
+        same ordering the posts feed renders thumbnails in."""
+        rows = (
+            await self.session.execute(
+                select(ImageRecord.id)
+                .where(ImageRecord.primary_post_id == post_id)
+                .order_by(ImageRecord.id.asc())
+            )
+        ).scalars().all()
+        return list(rows)
+
+    @staticmethod
+    def _stated_map(image_ids: list[int], start: int | None) -> dict | None:
+        """Per-image stated_page = start, start+1, ... when the post stated a
+        starting page; None when it didn't (fall back to capture order)."""
+        if start is None:
+            return None
+        return {iid: start + i for i, iid in enumerate(image_ids)}
+
+    async def promote_post_to_series(self, post_id: int) -> dict:
+        """Case 1: a self-contained multi-image post becomes its own series.
+        Creates a series tag named after the post and the post's images as its
+        flat pages (stated pages parsed from the post when present)."""
+        from .tag_service import TagService
+
+        post = await self.session.get(Post, post_id)
+        if post is None:
+            raise SeriesError(f"post {post_id} not found")
+        image_ids = await self._post_images_ordered(post_id)
+        if not image_ids:
+            raise SeriesError("post has no images to make a series from")
+        name = (post.post_title or f"Series from post {post_id}").strip()[:200]
+        tag = await TagService(self.session).find_or_create(name, TagKind.series)
+        rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}")
+        start = rng[0] if rng else None
+        added = await self.add_images(
+            tag.id, image_ids, stated_pages=self._stated_map(image_ids, start),
+        )
+        return {
+            "series_tag_id": tag.id, "name": tag.name, "added": added,
+        }
+
+    async def add_post(self, series_tag_id: int, post_id: int) -> dict:
+        """Case 2: STAGE a post's images as PENDING pages of an existing series
+        (#789 P2). They don't enter the ordered run yet — the operator drops junk
+        (text-free alts, bumpers) and places the keepers via place_pending, so
+        the series-global numbering stays clean. Stated pages are parsed from the
+        post when present."""
+        await self._require_series(series_tag_id)
+        post = await self.session.get(Post, post_id)
+        if post is None:
+            raise SeriesError(f"post {post_id} not found")
+        image_ids = await self._post_images_ordered(post_id)
+        if not image_ids:
+            raise SeriesError("post has no images to add")
+        # Skip images already in THIS series (placed or pending); move any in
+        # another series here as pending (image_id is UNIQUE).
+        existing = dict(
+            (
+                await self.session.execute(
+                    select(SeriesPage.image_id, SeriesPage.series_tag_id)
+                    .where(SeriesPage.image_id.in_(image_ids))
+                )
+            ).all()
+        )
+        to_stage = [i for i in image_ids if existing.get(i) != series_tag_id]
+        if not to_stage:
+            return {"series_tag_id": series_tag_id, "staged": 0}
+        await self.session.execute(
+            SeriesPage.__table__.delete().where(
+                SeriesPage.image_id.in_(to_stage)
+            )
+        )
+        # Store the post's parsed START page on every staged page (constant), so
+        # the group's start survives junk removal and seeds "Place from page N".
+        rng = parse_page_range(f"{post.post_title or ''} {post.description or ''}")
+        start = rng[0] if rng else None
+        await self.session.execute(
+            pg_insert(SeriesPage).values(
+                [
+                    {
+                        "series_tag_id": series_tag_id,
+                        "image_id": iid,
+                        "status": "pending",
+                        "page_number": None,
+                        "stated_page": start,
+                    }
+                    for iid in to_stage
+                ]
+            )
+        )
+        return {"series_tag_id": series_tag_id, "staged": len(to_stage)}
+
+    async def place_pending(
+        self,
+        series_tag_id: int,
+        image_ids: list[int],
+        start_page: int | None = None,
+    ) -> int:
+        """Place staged (pending) pages into the run, numbering them sequentially
+        from `start_page` in the given order (start_page, start_page+1, …). With
+        start_page None they're placed unnumbered (sorting to the tail). Drop the
+        junk BEFORE placing so the numbers line up; only pending pages are
+        touched, and other pages keep their numbers."""
+        await self._require_series(series_tag_id)
+        ids = self._clean_ids(image_ids)
+        rows = dict(
+            (
+                await self.session.execute(
+                    select(SeriesPage.image_id, SeriesPage.status).where(
+                        and_(
+                            SeriesPage.series_tag_id == series_tag_id,
+                            SeriesPage.image_id.in_(ids),
+                        )
+                    )
+                )
+            ).all()
+        )
+        # Preserve the caller's order, restricted to actually-pending pages.
+        staged = [i for i in ids if rows.get(i) == "pending"]
+        if not staged:
+            return 0
+        for offset, iid in enumerate(staged):
             await self.session.execute(
                 update(SeriesPage)
                 .where(
@@ -160,13 +652,151 @@ class SeriesService:
                         SeriesPage.image_id == iid,
                     )
                 )
-                .values(page_number=idx)
+                .values(
+                    status="placed",
+                    page_number=(
+                        start_page + offset if start_page is not None else None
+                    ),
+                )
+            )
+        return len(staged)
+
+    # ---- browse list (FC-6.2) --------------------------------------------
+
+    async def list_series(
+        self, *, sort: str = "recent", artist_id: int | None = None
+    ) -> list[dict]:
+        """Series cards for the browse view: cover thumb, name, artist, divider
+        + page counts, a missing-Part flag, and last-updated. sort ∈
+        recent|name|size."""
+        # Page counts + most-recent page activity per series.
+        page_rows = (
+            await self.session.execute(
+                select(
+                    SeriesPage.series_tag_id,
+                    func.count().label("pages"),
+                    func.max(SeriesPage.updated_at).label("updated_at"),
+                )
+                .where(SeriesPage.status == "placed")
+                .group_by(SeriesPage.series_tag_id)
+            )
+        ).all()
+        page_count = {r.series_tag_id: r.pages for r in page_rows}
+        updated_by_series = {r.series_tag_id: r.updated_at for r in page_rows}
+
+        # Dividers per series (for count + missing-Part gap), ordered by the
+        # global position of their anchor page.
+        drows = (
+            await self.session.execute(
+                select(
+                    SeriesChapter.series_tag_id,
+                    SeriesChapter.id,
+                    SeriesChapter.stated_part,
+                    SeriesPage.page_number,
+                )
+                .join(SeriesPage, SeriesPage.id == SeriesChapter.anchor_page_id)
+                .order_by(
+                    SeriesChapter.series_tag_id, SeriesPage.page_number.asc()
+                )
+            )
+        ).all()
+        dividers_by_series: dict[int, list] = {}
+        for r in drows:
+            dividers_by_series.setdefault(r.series_tag_id, []).append(r)
+
+        # Cover = the first page (min page_number) of each series.
+        cover_q = (
+            select(
+                SeriesPage.series_tag_id,
+                ImageRecord.sha256,
+                ImageRecord.mime,
+                ImageRecord.thumbnail_path,
+                ImageRecord.artist_id,
+                func.row_number()
+                .over(
+                    partition_by=SeriesPage.series_tag_id,
+                    order_by=(
+                        SeriesPage.page_number.asc(), SeriesPage.id.asc()
+                    ),
+                )
+                .label("rn"),
+            )
+            .join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
+            .where(SeriesPage.status == "placed")
+            .subquery()
+        )
+        cover_rows = (
+            await self.session.execute(
+                select(
+                    cover_q.c.series_tag_id,
+                    cover_q.c.sha256,
+                    cover_q.c.mime,
+                    cover_q.c.thumbnail_path,
+                    cover_q.c.artist_id,
+                ).where(cover_q.c.rn == 1)
+            )
+        ).all()
+        cover_by_series = {r.series_tag_id: r for r in cover_rows}
+
+        # Artist names for the covers.
+        artist_ids = {
+            r.artist_id for r in cover_rows if r.artist_id is not None
+        }
+        artist_by_id: dict[int, object] = {}
+        if artist_ids:
+            arows = (
+                await self.session.execute(
+                    select(Artist.id, Artist.name, Artist.slug)
+                    .where(Artist.id.in_(artist_ids))
+                )
+            ).all()
+            artist_by_id = {a.id: a for a in arows}
+
+        tags = (
+            await self.session.execute(
+                select(Tag.id, Tag.name).where(Tag.kind == TagKind.series)
+            )
+        ).all()
+
+        out: list[dict] = []
+        for t in tags:
+            cover = cover_by_series.get(t.id)
+            if artist_id is not None and (
+                cover is None or cover.artist_id != artist_id
+            ):
+                continue
+            divs = dividers_by_series.get(t.id, [])
+            gap = bool(
+                self._part_gaps(
+                    [{"id": d.id, "stated_part": d.stated_part} for d in divs]
+                )
+            )
+            artist = artist_by_id.get(cover.artist_id) if cover else None
+            updated = updated_by_series.get(t.id)
+            out.append(
+                {
+                    "id": t.id,
+                    "name": t.name,
+                    "cover_thumbnail_url": (
+                        thumbnail_url(
+                            cover.thumbnail_path, cover.sha256, cover.mime
+                        )
+                        if cover
+                        else None
+                    ),
+                    "artist_name": artist.name if artist else None,
+                    "artist_slug": artist.slug if artist else None,
+                    "chapter_count": len(divs),
+                    "page_count": page_count.get(t.id, 0),
+                    "has_gap": gap,
+                    "updated_at": updated.isoformat() if updated else None,
+                }
             )
 
-    async def set_cover(self, series_tag_id: int, image_id: int) -> None:
-        await self._require_series(series_tag_id)
-        order = await self._member_order(series_tag_id)
-        if image_id not in order:
-            raise SeriesError(f"image {image_id} is not in this series")
-        new_order = [image_id] + [i for i in order if i != image_id]
-        await self.reorder(series_tag_id, new_order)
+        if sort == "name":
+            out.sort(key=lambda s: s["name"].lower())
+        elif sort == "size":
+            out.sort(key=lambda s: s["page_count"], reverse=True)
+        else:  # recent
+            out.sort(key=lambda s: s["updated_at"] or "", reverse=True)
+        return out

backend/app/services/showcase_service.py

@@ -20,11 +20,20 @@ class ShowcaseService:
     async def random_sample(self, limit: int = 60) -> list[dict]:
         if limit < 1 or limit > 200:
             raise ValueError("limit must be between 1 and 200")
+        # Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows
+        # from each sampled page, so sequentially-imported near-duplicates
+        # (multi-image posts, variant sets) come back adjacent and cluster in the
+        # showcase ("three near-identical in a row"). Sampling a multiple of
+        # `limit` spans more pages, and ORDER BY random() before taking `limit`
+        # breaks the physical adjacency — far better spread, still cheap
+        # (random() over a few hundred rows, not the whole table).
+        oversample = min(limit * 5, 1000)
         stmt = select(ImageRecord).from_statement(
             text(
-                "SELECT * FROM image_record "
-                "TABLESAMPLE SYSTEM_ROWS(:n)"
-            ).bindparams(n=limit)
+                "SELECT * FROM ("
+                "  SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)"
+                ") sub ORDER BY random() LIMIT :n"
+            ).bindparams(o=oversample, n=limit)
         )
         rows = (await self.session.execute(stmt)).scalars().all()
         return [

backend/app/services/source_service.py

@@ -64,6 +64,19 @@ class SourceRecord:
     consecutive_failures: int
     next_check_at: str | None
     backfill_runs_remaining: int
+    # plan #693: derived from config_overrides for the UI badge.
+    backfill_state: str | None  # "running" | "complete" | "stalled" | None (idle)
+    backfill_chunks: int
+    # plan #697: a running deep-walk that bypasses the Patreon seen-ledger
+    # (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
+    backfill_bypass_seen: bool
+    # #830: a running deep-walk in RECAPTURE mode (re-grab post bodies/links +
+    # localize on-disk inline images, no media re-download). Lets the badge label
+    # it "Recapturing".
+    backfill_recapture: bool
+    # plan #704: cumulative posts processed across the walk's chunks — live
+    # progress for the badge.
+    backfill_posts: int
 
     def to_dict(self) -> dict:
         return {
@@ -82,6 +95,11 @@ class SourceRecord:
             "consecutive_failures": self.consecutive_failures,
             "next_check_at": self.next_check_at,
             "backfill_runs_remaining": self.backfill_runs_remaining,
+            "backfill_state": self.backfill_state,
+            "backfill_chunks": self.backfill_chunks,
+            "backfill_bypass_seen": self.backfill_bypass_seen,
+            "backfill_recapture": self.backfill_recapture,
+            "backfill_posts": self.backfill_posts,
         }
 
 
@@ -89,10 +107,13 @@ class SourceRecord:
 
 _EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
 
-# Plan #544 follow-up: newly created enabled sources pre-arm backfill so
-# their first N polls walk gallery-dl's full post history with the longer
-# timeout (matches the manual "Deep scan" button's default).
-NEW_SOURCE_BACKFILL_RUNS = 3
+# Plan #693: backfill safety cap. "Start backfill" (and a newly created
+# enabled source) arms a run-until-done walk; this caps how many time-boxed
+# chunks it may spend before pausing as "stalled", so a pathological walk that
+# never reaches the bottom can't run forever. Generous on purpose — at
+# BACKFILL_CHUNK_SECONDS (600s) per chunk this is ~33h of cumulative walk, far
+# beyond any real catalog; the cursor stall-guard is the real terminator.
+BACKFILL_MAX_CHUNKS = 200
 
 
 class SourceService:
@@ -135,6 +156,7 @@ class SourceService:
         self, source: Source, artist: Artist, settings: ImportSettings,
     ) -> SourceRecord:
         nxt = compute_next_check_at(source, artist, settings)
+        co = source.config_overrides or {}
         return SourceRecord(
             id=source.id,
             artist_id=source.artist_id,
@@ -151,6 +173,11 @@ class SourceService:
             consecutive_failures=source.consecutive_failures or 0,
             next_check_at=nxt.isoformat() if nxt else None,
             backfill_runs_remaining=source.backfill_runs_remaining or 0,
+            backfill_state=co.get("_backfill_state"),
+            backfill_chunks=int(co.get("_backfill_chunks", 0)),
+            backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
+            backfill_recapture=bool(co.get("_backfill_recapture")),
+            backfill_posts=int(co.get("_backfill_posts", 0)),
         )
 
     async def _row_to_record(self, source: Source) -> SourceRecord:
@@ -212,16 +239,17 @@ class SourceService:
             select(func.count(Source.id)).where(Source.artist_id == artist_id)
         )).scalar_one()
 
-        # Plan #544 follow-up: a freshly added subscription has no archive
-        # yet, so the first few polls would walk the full post history in
-        # tick mode and trip exit:20 after ~20 contiguous archive hits —
-        # except there are none yet, so tick mode would walk forever and
-        # blow the wall-clock cap. Pre-arm backfill so the initial syncs
-        # use the longer timeout + skip:True walk. Tick mode resumes once
-        # the budget is spent or the queue drains.
-        # Disabled sources (incl. sidecar synthetics, url='sidecar:...')
-        # are never polled, so leave their counter at 0.
-        backfill_runs = NEW_SOURCE_BACKFILL_RUNS if enabled else 0
+        # Plan #693: a freshly added subscription has no archive yet, so it
+        # should walk its full post history once. Arm run-until-done backfill
+        # (state="running" + the chunk cap); the time-boxed chunks march to the
+        # bottom across ticks, then flip to "complete" and tick mode takes over.
+        # Disabled sources (incl. sidecar synthetics, url='sidecar:...') are
+        # never polled, so leave them idle.
+        if enabled:
+            config_overrides = {**(config_overrides or {}), "_backfill_state": "running"}
+            backfill_runs = BACKFILL_MAX_CHUNKS
+        else:
+            backfill_runs = 0
         source = Source(
             artist_id=artist_id, platform=platform, url=url,
             enabled=enabled, config_overrides=config_overrides,
@@ -286,22 +314,100 @@ class SourceService:
         await self.session.commit()
         return await self._row_to_record(source)
 
-    async def set_backfill_runs(
-        self, source_id: int, runs: int,
-    ) -> SourceRecord:
-        """Plan #544: arm a source for backfill mode. The next `runs`
-        download runs will use gallery-dl's full-walk config (skip: True
-        + 30-min timeout) instead of the catch-up default. Runs must be
-        1..10 — bigger is rejected to keep the operator from accidentally
-        setting a runaway budget."""
-        if not isinstance(runs, int) or runs < 1 or runs > 10:
-            raise ValueError("runs must be an integer in [1, 10]")
+    async def start_backfill(self, source_id: int) -> SourceRecord:
+        """Plan #693: arm a run-until-done backfill. Sets state="running" and
+        the chunk cap; download runs then walk the full post history in
+        time-boxed chunks (skip:True + BACKFILL_CHUNK_SECONDS), resuming from
+        the cursor each chunk, until gallery-dl reaches the bottom (→ state
+        "complete") or the cap/stall-guard pauses it (→ "stalled"). Clears any
+        prior cursor/chunk/stall state so a re-start walks fresh from the top."""
         source = (await self.session.execute(
             select(Source).where(Source.id == source_id)
         )).scalar_one_or_none()
         if source is None:
             raise LookupError(f"source id={source_id} not found")
-        source.backfill_runs_remaining = runs
+        co = dict(source.config_overrides or {})
+        co["_backfill_state"] = "running"
+        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
+                  "_backfill_posts"):
+            co.pop(k, None)
+        source.config_overrides = co
+        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
+        await self.session.commit()
+        return await self._row_to_record(source)
+
+    async def start_recovery(self, source_id: int) -> SourceRecord:
+        """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon
+        seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched
+        and re-evaluated under the CURRENT pHash threshold (tier-2 disk still
+        spares files we kept). Reuses the entire #693 backfill state machine
+        (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the
+        `_backfill_bypass_seen` flag that flips download mode to recovery. Clears
+        any prior cursor/chunk/stall state so it walks fresh from the top. The
+        flag is cleared on completion (download_service) and on stop.
+
+        Recovery is Patreon-only (the seen-ledger is Patreon's); for other
+        platforms the flag is inert (download_service ignores it) and the walk
+        runs as a plain backfill. The UI gates the action to Patreon sources."""
+        source = (await self.session.execute(
+            select(Source).where(Source.id == source_id)
+        )).scalar_one_or_none()
+        if source is None:
+            raise LookupError(f"source id={source_id} not found")
+        co = dict(source.config_overrides or {})
+        co["_backfill_state"] = "running"
+        co["_backfill_bypass_seen"] = True
+        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
+                  "_backfill_posts"):
+            co.pop(k, None)
+        source.config_overrides = co
+        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
+        await self.session.commit()
+        return await self._row_to_record(source)
+
+    async def start_recapture(self, source_id: int) -> SourceRecord:
+        """#830: arm a RECAPTURE walk — a backfill that re-grabs EVERY post's body
+        + external links (detail-fetching empty bodies) and localizes already-on-
+        disk inline images, WITHOUT re-downloading media. Reuses the entire #693
+        backfill state machine plus a `_backfill_recapture` flag that flips
+        download mode to recapture. Distinct from recovery (which re-downloads the
+        whole source); the two flags are mutually exclusive, so arming recapture
+        clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
+        fresh from the top. The flag is cleared on completion (download_service)
+        and on stop. Recapture is Patreon-only (the native ingester's post-record
+        capture); inert elsewhere. The UI gates the action to Patreon sources."""
+        source = (await self.session.execute(
+            select(Source).where(Source.id == source_id)
+        )).scalar_one_or_none()
+        if source is None:
+            raise LookupError(f"source id={source_id} not found")
+        co = dict(source.config_overrides or {})
+        co["_backfill_state"] = "running"
+        co["_backfill_recapture"] = True
+        co.pop("_backfill_bypass_seen", None)  # mutually exclusive with recovery
+        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
+                  "_backfill_posts"):
+            co.pop(k, None)
+        source.config_overrides = co
+        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
+        await self.session.commit()
+        return await self._row_to_record(source)
+
+    async def stop_backfill(self, source_id: int) -> SourceRecord:
+        """Plan #693: cancel an in-progress backfill — back to idle/tick mode.
+        Clears the running state + cursor/chunk/stall bookkeeping."""
+        source = (await self.session.execute(
+            select(Source).where(Source.id == source_id)
+        )).scalar_one_or_none()
+        if source is None:
+            raise LookupError(f"source id={source_id} not found")
+        co = dict(source.config_overrides or {})
+        for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
+                  "_backfill_chunks", "_backfill_bypass_seen", "_backfill_recapture",
+                  "_backfill_posts"):
+            co.pop(k, None)
+        source.config_overrides = co
+        source.backfill_runs_remaining = 0
         await self.session.commit()
         return await self._row_to_record(source)
 

backend/app/services/subscribestar_client.py

@@ -0,0 +1,602 @@
+"""Native SubscribeStar client — the SubscribeStar adapter's read path.
+
+Unlike Patreon (a clean JSON:API), SubscribeStar has NO public API: gallery-dl
+scrapes its HTML, and so do we. The platform-agnostic core (`ingest_core`) only
+calls `client.iter_posts` / `extract_media` / the optional seams, so the
+HTML-scrape divergence is contained entirely to this module.
+
+Feed shape (characterized from a live sample 2026-06-17 — Scribe note
+"SubscribeStar HTML characterization"):
+  - Page 1 is the creator page HTML: GET <base>/<slug>.
+  - Each page embeds `data-role="infinite_scroll-next_page" href="/posts?...&page=N
+    &slug=<slug>&sort_by=newest"`. That path returns JSON {"html": "<...posts...>"};
+    the fragment carries the NEXT page link, until it runs out.
+  - A post is `<div class="post is-shown ..." data-id="<post_id>" ...>`; body lives
+    in `.post-content .trix-content`; date in `.post-date`; media in a `data-gallery`
+    JSON manifest on `.uploads-images`.
+
+`campaign_id` for SubscribeStar is the full creator URL (e.g.
+https://subscribestar.adult/sabu) — the host (.com vs .adult) and slug both come
+from it, so no separate resolver is needed.
+
+Drift is loud on purpose (mirrors patreon_client): an HTML login/age-gate where
+the feed was expected is AUTH (rotate cookies); a feed whose post structure we
+can't parse at all is DRIFT (the scraper needs updating). FC runs on a
+plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime
+from html import unescape
+from pathlib import Path
+from urllib.parse import urljoin, urlsplit
+
+import requests
+
+from ..utils.paths import filehash_from_url
+from .native_ingest_common import (
+    _MAX_429_RETRIES,
+    NativeAuthError,
+    NativeDriftError,
+    NativeIngestError,
+    basename_from_url,
+    make_session,
+    retry_after_seconds,
+)
+
+log = logging.getLogger(__name__)
+
+_TIMEOUT_SECONDS = 30.0
+# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven
+# way to read SubscribeStar without tripping its /verify_subscriber gate. In that
+# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*,
+# Accept-Language, and a same-site Referer (root/) on EVERY request — including
+# the first creator-page GET — and uses NO X-Requested-With on either the creator
+# page or the "load more" JSON endpoint (it GETs and parses the body as JSON).
+# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a
+# browser that SubscribeStar 302'd the adult-creator page to /<slug>/verify_
+# subscriber even with valid cookies (cheunart, 2026-06-17).
+_FIREFOX_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0"
+)
+_GDL_HEADERS = {
+    "User-Agent": _FIREFOX_UA,
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+# A post block opens with this wrapper; we slice the page between consecutive
+# occurrences (regex can't match the balanced close of nested divs, so chunk-
+# per-post is the robust approach gallery-dl also uses).
+#
+# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
+# container regardless of the classes that follow), exactly what gallery-dl's
+# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
+# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
+# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
+# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
+# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
+# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
+_POST_OPEN = '<div class="post '
+_POST_ID_RE = re.compile(r'data-id="(\d+)"')
+# The date may be plain text OR wrapped in an <a> permalink — image posts wrap it
+# (`<div class="post-date"><a href="/posts/ID">DATE</a></div>`), text-only posts
+# don't. gallery-dl's `_data_from_post` handles both: take text up to the first
+# `</`, then whatever follows the last `>`. A naive
+# `<div class="post-date">([^<]+)</div>` regex matched ONLY the unwrapped case, so
+# every image post got a null date and sorted to the top (cheunart 2026-06-17).
+_DATE_OPEN = 'class="post-date">'
+# The body lives between these two LITERAL markers — exactly gallery-dl's
+# `_data_from_post` extraction: from the post_content-text wrapper open to the
+# youtube-uploads div that always follows post-content. A balanced-</div> regex
+# either returned empty or over-captured into sibling upload divs AND the
+# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
+# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
+# `<html><body>…</body></html>` document, so strip to the body inner when present.
+_CONTENT_OPEN = '<div class="post-content" data-role="post_content-text">'
+_CONTENT_CLOSE = '</div><div class="post-uploads for-youtube"'
+_GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
+# Document + audio attachments are NOT in data-gallery — gallery-dl's
+# _media_from_post scrapes them from their own `uploads-docs` / `uploads-audios`
+# sections, splitting on each preview block. Some posts deliver content ONLY
+# through these (PDFs/zips, audio), so we must walk them too.
+_DOC_SPLIT_RE = re.compile(r'class="doc_preview[" ]')
+_AUDIO_SPLIT_RE = re.compile(r'class="audio_preview-data[" ]')
+_NEXT_PAGE_RE = re.compile(
+    r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
+)
+# Signals an auth/age wall served in place of the feed (cookies expired or the
+# age cookie missing) rather than a real — possibly empty — creator feed.
+_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
+
+# An interstitial served INSTEAD of the feed — characterized so a drift error
+# reports the actual cause (bot challenge / age gate / login) rather than a bare
+# "markup changed". (name, substrings to look for, case-insensitive.)
+_INTERSTITIAL_MARKERS = (
+    ("cloudflare/bot-challenge", (
+        "just a moment", "cf-challenge", "challenge-platform", "cf_chl",
+        "attention required", "enable javascript and cookies",
+    )),
+    ("age-gate", (
+        "18 or older", "adult content", "age_confirmation", "i am over",
+        "confirm your age", "must be 18",
+    )),
+    ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
+    ("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
+)
+
+
+def _describe_page(html: str) -> str:
+    """A short, log-safe description of an unexpected page: its <title> + which
+    known interstitial it resembles (bot challenge / age gate / login / captcha)."""
+    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
+    low = html.lower()
+    hits = [name for name, needles in _INTERSTITIAL_MARKERS
+            if any(n.lower() in low for n in needles)]
+    return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
+
+
+class SubscribeStarAPIError(NativeIngestError):
+    """Base for native SubscribeStar client failures. status_code / retry_after
+    are inherited from NativeIngestError."""
+
+
+class SubscribeStarAuthError(SubscribeStarAPIError, NativeAuthError):
+    """Auth/authorization failure — expired cookies, missing age cookie, or an
+    HTML login/age wall served where the feed was expected. Fix = rotate the
+    credential, not update the scraper. Maps to error_type 'auth_error'."""
+
+
+class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
+    """The feed HTML did not match the structure we scrape (no recognizable post
+    blocks AND no known empty-feed state). The scrape analog of API drift — fail
+    loud so the import step flags 'SubscribeStar changed its markup' instead of
+    silently importing nothing."""
+
+
+@dataclass
+class MediaItem:
+    """One resolved downloadable item belonging to a SubscribeStar post.
+
+    Fields mirror patreon_client.MediaItem (so the downloader is structurally the
+    same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
+    full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
+    so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
+    """
+
+    url: str
+    filename: str
+    kind: str
+    filehash: str | None
+    post_id: str
+    media_id: str
+
+
+def _extr(text: str, start: str, end: str) -> str:
+    """Substring between the first `start` and the next `end` after it (gallery-
+    dl's `text.extr`); '' when either marker is absent."""
+    i = text.find(start)
+    if i < 0:
+        return ""
+    i += len(start)
+    j = text.find(end, i)
+    if j < 0:
+        return ""
+    return text[i:j]
+
+
+def _extract_content(chunk: str) -> str:
+    """The post body HTML — gallery-dl's `_data_from_post` content rule: between
+    the post_content-text wrapper and the youtube-uploads div, with the trix
+    editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
+    content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
+    if "<html><body>" in content:
+        content = _extr(content, "<body>", "</body>")
+    return content.strip()
+
+
+def _attachment_item(
+    frag: str, base: str, post_id: str, *, kind: str, title_marker: str, url_attr: str
+) -> MediaItem | None:
+    """One doc/audio attachment from its preview block — gallery-dl's
+    `_media_from_post` attachment/audio fields. `url_attr` is the URL-bearing
+    attribute (`href="` for docs, `src="` for audio); `title_marker` precedes the
+    display name. Returns None when the block carries no URL."""
+    rel = unescape(_extr(frag, url_attr, '"'))
+    if not rel:
+        return None
+    url = urljoin(base + "/", rel)
+    upload_id = _extr(frag, 'data-upload-id="', '"')
+    name = unescape(_extr(frag, title_marker, "<")).strip()
+    return MediaItem(
+        url=url,
+        filename=name or basename_from_url(url),
+        kind=kind,
+        filehash=filehash_from_url(url),
+        post_id=post_id,
+        media_id=str(upload_id or ""),
+    )
+
+
+def _split_creator_url(campaign_id: str) -> tuple[str, str]:
+    """`campaign_id` is the creator URL → (base, slug).
+
+    base = scheme://host (preserving .com vs .adult); slug = first path segment.
+    """
+    parts = urlsplit(campaign_id)
+    base = f"{parts.scheme or 'https'}://{parts.netloc}"
+    slug = parts.path.strip("/").split("/")[0] if parts.path else ""
+    return base, slug
+
+
+def _parse_ss_datetime(text: str) -> str | None:
+    """SubscribeStar renders human dates: 'Jun 17, 2026 03:19 am' (and an
+    'Updated on <date>' variant). Return ISO-8601 (UTC-naive) or None."""
+    s = unescape(text or "").strip()
+    if s.lower().startswith("updated on "):
+        s = s[len("updated on "):].strip()
+    for fmt in ("%b %d, %Y %I:%M %p", "%b %d, %Y"):
+        try:
+            return datetime.strptime(s, fmt).isoformat()
+        except ValueError:
+            continue
+    return None
+
+
+class SubscribeStarClient:
+    """Synchronous SubscribeStar HTML-scrape read client. Construct with a path
+    to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
+    materializes, already carrying the age cookie via augment_cookies)."""
+
+    def __init__(
+        self,
+        cookies_path: str | Path | None,
+        *,
+        request_sleep: float = 0.0,
+        max_retries: int = _MAX_429_RETRIES,
+    ):
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        # gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept-
+        # Language; Referer is stamped per-walk once the creator base is known).
+        self._session = make_session(
+            cookies_path, accept="*/*", extra_headers=_GDL_HEADERS
+        )
+        self._request_sleep = request_sleep or 0.0
+        self._max_retries = max_retries
+
+    # -- request -----------------------------------------------------------
+
+    def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
+        if self._request_sleep > 0:
+            time.sleep(self._request_sleep)
+        attempt = 0
+        while True:
+            try:
+                resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
+            except requests.RequestException as exc:
+                raise SubscribeStarAPIError(
+                    f"SubscribeStar request failed ({url}): {exc}"
+                ) from exc
+            if resp.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
+                delay = retry_after_seconds(resp, attempt)
+                log.warning(
+                    "SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
+                    url, delay, attempt, self._max_retries,
+                )
+                time.sleep(delay)
+                continue
+            break
+        # gallery-dl's own gating signal: SubscribeStar 302-redirects an
+        # unauthenticated / age-unconfirmed request to /verify_subscriber or
+        # /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
+        # not drift — the fix is to rotate cookies / set the age cookie.
+        if resp.history and (
+            "/verify_subscriber" in resp.url
+            or "/age_confirmation_warning" in resp.url
+        ):
+            raise SubscribeStarAuthError(
+                f"SubscribeStar redirected to {resp.url} — auth/age wall "
+                f"(rotate cookies or set the 18+ age cookie; requested {url})",
+                status_code=resp.status_code,
+            )
+        if resp.status_code in (401, 403):
+            raise SubscribeStarAuthError(
+                f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
+                f"(cookies expired or tier insufficient; {url})",
+                status_code=resp.status_code,
+            )
+        if resp.status_code != 200:
+            retry_after = None
+            if resp.status_code == 429:
+                hdr = resp.headers.get("Retry-After")
+                if hdr:
+                    try:
+                        retry_after = float(hdr)
+                    except (TypeError, ValueError):
+                        retry_after = None
+            raise SubscribeStarAPIError(
+                f"SubscribeStar returned HTTP {resp.status_code} ({url})",
+                status_code=resp.status_code,
+                retry_after=retry_after,
+            )
+        return resp
+
+    def _feed_html(self, url: str) -> str:
+        """Page 1: the creator page (full HTML document)."""
+        resp = self._get(url)
+        text = resp.text or ""
+        if any(m in text for m in _LOGIN_MARKERS) and _POST_OPEN not in text:
+            raise SubscribeStarAuthError(
+                f"SubscribeStar served a login/age wall instead of the feed "
+                f"(cookies expired or age cookie missing; {url})"
+            )
+        return text
+
+    def _loadmore_html(self, url: str) -> str:
+        """Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the
+        `/posts?...` endpoint, which returns JSON {"html": "..."}."""
+        resp = self._get(url)
+        try:
+            payload = resp.json()
+        except ValueError as exc:
+            raise SubscribeStarAuthError(
+                f"SubscribeStar 'load more' returned non-JSON (session expired?; "
+                f"{url}): {exc}"
+            ) from exc
+        html = payload.get("html") if isinstance(payload, dict) else None
+        return html if isinstance(html, str) else ""
+
+    # -- parsing -----------------------------------------------------------
+
+    @staticmethod
+    def _post_chunks(html: str) -> list[str]:
+        """Slice the page into one chunk per post (between consecutive wrapper
+        opens). Regex can't match a post's balanced close, so each chunk runs to
+        the next post's open (or end of fragment) — enough to scope per-post
+        field extraction."""
+        starts = [m.start() for m in re.finditer(re.escape(_POST_OPEN), html)]
+        chunks = []
+        for i, start in enumerate(starts):
+            end = starts[i + 1] if i + 1 < len(starts) else len(html)
+            chunks.append(html[start:end])
+        return chunks
+
+    def _parse_post(self, chunk: str) -> dict | None:
+        m = _POST_ID_RE.search(chunk)
+        if not m:
+            return None
+        post_id = m.group(1)
+        # gallery-dl date method: text up to first '</', then after the last '>'
+        # — handles both plain and <a>-wrapped dates (see _DATE_OPEN comment).
+        raw_date = _extr(chunk, _DATE_OPEN, "</").rpartition(">")[2]
+        published = _parse_ss_datetime(raw_date) if raw_date else None
+        content = _extract_content(chunk)
+        return {
+            "id": post_id,
+            "attributes": {
+                # SubscribeStar has no title field; the importer synthesizes a
+                # display title from the body's first line (sidecar util).
+                "title": "",
+                "content": content,
+                "published_at": published,
+                "post_type": "subscribestar",
+            },
+            # Raw chunk retained so extract_media parses the data-gallery manifest
+            # and post_is_gated can scan for the locked-teaser marker.
+            "_html": chunk,
+        }
+
+    def _parse_posts(self, html: str) -> list[dict]:
+        posts = []
+        for chunk in self._post_chunks(html):
+            post = self._parse_post(chunk)
+            if post is not None:
+                posts.append(post)
+        if posts:
+            dated = sum(1 for p in posts if p["attributes"].get("published_at"))
+            bodied = sum(1 for p in posts if p["attributes"].get("content"))
+            log.info(
+                "SubscribeStar parsed %d posts (%d dated, %d with body)",
+                len(posts), dated, bodied,
+            )
+            # Canary for this exact failure class: posts parsed but NONE got a
+            # date or a body, while the raw markers ARE present → our extraction
+            # diverged from the live markup (e.g. the <a>-wrapped date bug). Log
+            # the marker counts so the cause is diagnosable from the worker log
+            # alone, without re-fetching the authed page.
+            if dated == 0 or bodied == 0:
+                log.warning(
+                    "SubscribeStar parse canary: %d posts but dated=%d bodied=%d; "
+                    "raw markers post-date=%d post_content-text=%d data-gallery=%d "
+                    "— extraction likely diverged from the live markup",
+                    len(posts), dated, bodied,
+                    html.count('class="post-date"'),
+                    html.count("post_content-text"),
+                    html.count("data-gallery"),
+                )
+        return posts
+
+    @staticmethod
+    def _next_page_href(html: str) -> str | None:
+        m = _NEXT_PAGE_RE.search(html)
+        return unescape(m.group(1)) if m else None
+
+    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
+        """Resolve downloadable media (gallery-dl's `_media_from_post`): the
+        per-post `data-gallery` JSON manifest (images/videos), PLUS document
+        attachments (`uploads-docs`) and audio (`uploads-audios`) — some posts
+        deliver content only through the latter two. `/previews` (locked teaser)
+        gallery items are skipped. `included_index` is unused (media is inline)."""
+        chunk = post.get("_html") or ""
+        base = post.get("_base") or "https://www.subscribestar.com"
+        post_id = str(post.get("id") or "")
+        items: list[MediaItem] = []
+        for gm in _GALLERY_RE.finditer(chunk):
+            try:
+                gallery = json.loads(unescape(gm.group(1)))
+            except (ValueError, TypeError):
+                continue
+            if not isinstance(gallery, list):
+                continue
+            for it in gallery:
+                if not isinstance(it, dict):
+                    continue
+                rel = it.get("url")
+                if not isinstance(rel, str) or not rel:
+                    continue
+                # gallery-dl's _media_from_post: a gallery item whose URL is under
+                # /previews is a locked/blurred TEASER, not the real file — skip it
+                # (the SubscribeStar analog of the Patreon gated-preview bug #874).
+                # This is why a locked post yields no downloadable media.
+                if "/previews" in rel:
+                    continue
+                url = urljoin(base + "/", rel)
+                media_id = str(it.get("id") or "")
+                name = it.get("original_filename")
+                filename = name if isinstance(name, str) and name else basename_from_url(url)
+                items.append(
+                    MediaItem(
+                        url=url,
+                        filename=filename,
+                        kind=str(it.get("type") or "image"),
+                        filehash=filehash_from_url(url),
+                        post_id=post_id,
+                        media_id=media_id,
+                    )
+                )
+        # Document attachments (uploads-docs → doc_preview blocks): href URL.
+        docs = _extr(chunk, 'class="uploads-docs"', 'class="post-edit_form"')
+        for frag in _DOC_SPLIT_RE.split(docs)[1:]:
+            item = _attachment_item(
+                frag, base, post_id, kind="attachment",
+                title_marker='doc_preview-title">', url_attr='href="',
+            )
+            if item is not None:
+                items.append(item)
+        # Audio attachments (uploads-audios → audio_preview-data blocks): src URL.
+        audios = _extr(chunk, 'class="uploads-audios"', 'class="post-edit_form"')
+        for frag in _AUDIO_SPLIT_RE.split(audios)[1:]:
+            item = _attachment_item(
+                frag, base, post_id, kind="audio",
+                title_marker='audio_preview-title">', url_attr='src="',
+            )
+            if item is not None:
+                items.append(item)
+        return items
+
+    @staticmethod
+    def post_meta(post: dict) -> dict:
+        """Title + date for the preview sample. Title is synthesized from the body
+        (SubscribeStar has no title field)."""
+        attrs = post.get("attributes") or {}
+        return {"title": None, "date": attrs.get("published_at")}
+
+    @staticmethod
+    def post_is_gated(post: dict) -> bool:
+        """True when the subscriber cannot view this post (locked teaser). #874
+        "no stub for gated content": the core skips it ENTIRELY (no media, no
+        post-record), so we don't replicate a hollow teaser in Curator.
+
+        SubscribeStar does NOT expose downloadable preview media for locked posts
+        (no `data-gallery`), so the Patreon "downloaded blurred junk" failure
+        can't happen here — this gate only prevents capturing an empty teaser
+        stub. Detect the locked marker conservatively (default to NOT gated when
+        absent, so we never over-filter an accessible text post). The exact marker
+        is being confirmed against a live locked sample; until then we gate only on
+        the explicit lock-overlay class SubscribeStar renders on a paywalled post.
+        """
+        chunk = post.get("_html") or ""
+        return 'class="post-content-locked"' in chunk or "for-locked_content" in chunk
+
+    @staticmethod
+    def post_record_key(post: dict) -> tuple[str, str] | None:
+        """`(ledger_key, post_id)` for a post's seen-ledger entry (the `post:<id>`
+        synthetic key gates post-record capture through the same ledger as media),
+        or None when the post has no id."""
+        pid = post.get("id")
+        pid = str(pid) if pid is not None else ""
+        if not pid:
+            return None
+        return (f"post:{pid}", pid)
+
+    # -- iteration ---------------------------------------------------------
+
+    def iter_posts(
+        self, campaign_id: str, cursor: str | None = None
+    ) -> Iterator[tuple[dict, dict, str | None]]:
+        """Yield (post, {}, page_cursor) for every post in the feed.
+
+        `campaign_id` is the creator URL. `cursor` is the relative "load more"
+        href that fetches a page (None → page 1, the creator page HTML). The
+        yielded `page_cursor` is the href that FETCHED this post's page, so the
+        core checkpoints a value that re-fetches the same page on resume (matching
+        the Patreon cursor contract).
+        """
+        base, slug = _split_creator_url(campaign_id)
+        if not slug:
+            raise SubscribeStarDriftError(
+                f"Could not extract a creator slug from {campaign_id!r}"
+            )
+        # Same-site Referer (gallery-dl sends root/ on every request).
+        self._session.headers["Referer"] = f"{base}/"
+        current = cursor
+        first_page = True
+        while True:
+            page_cursor = current
+            if current is None:
+                html = self._feed_html(f"{base}/{slug}")
+            else:
+                html = self._loadmore_html(urljoin(base + "/", current))
+            posts = self._parse_posts(html)
+            if first_page and not posts and _POST_OPEN not in html:
+                # Page 1 with no recognizable post wrappers at all: either a brand
+                # new creator with zero posts, or our scraper is stale. The core's
+                # body canary catches systematic emptiness across a populated feed;
+                # here we only raise if the feed container itself is missing.
+                if 'data-role="posts_container-list"' not in html:
+                    # Report what we actually got — length + the page's identity —
+                    # so a served interstitial (bot challenge / age gate / login)
+                    # is named instead of mislabeled "markup changed".
+                    looks_json = html.lstrip()[:1] in ("{", "[")
+                    raise SubscribeStarDriftError(
+                        f"SubscribeStar feed for {slug!r} had no posts and no "
+                        f"recognizable feed container ({len(html)} bytes"
+                        f"{', looks like JSON' if looks_json else ''}; "
+                        f"{_describe_page(html)})"
+                    )
+            for post in posts:
+                post["_base"] = base
+                yield post, {}, page_cursor
+            next_href = self._next_page_href(html)
+            if not next_href:
+                return
+            current = next_href
+            first_page = False
+
+    # -- verify ------------------------------------------------------------
+
+    def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
+        """Cheap auth probe: fetch the first feed page and report whether the
+        credential authenticated, without downloading anything."""
+        base, slug = _split_creator_url(campaign_id)
+        if not slug:
+            return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
+        self._session.headers["Referer"] = f"{base}/"
+        try:
+            html = self._feed_html(f"{base}/{slug}")
+        except SubscribeStarAuthError as exc:
+            return False, f"SubscribeStar rejected the credential — {exc}"
+        except SubscribeStarAPIError as exc:
+            return None, f"Couldn't verify (network/HTTP issue): {exc}"
+        if _POST_OPEN in html or 'data-role="posts_container-list"' in html:
+            return True, "Credentials valid — the SubscribeStar feed loaded."
+        return None, "Couldn't verify — SubscribeStar feed shape unrecognized."

backend/app/services/subscribestar_downloader.py

@@ -0,0 +1,206 @@
+"""Native SubscribeStar media downloader — the SubscribeStar counterpart to
+patreon_downloader.
+
+Given a SubscribeStar post and its resolved `MediaItem`s
+(subscribestar_client.extract_media), download the media to gallery-dl's on-disk
+layout (so existing gallery-dl downloads are recognized on disk and not
+re-fetched at cutover), write the post-first sidecars the importer consumes, and
+report per-media outcomes.
+
+Simpler than the Patreon downloader: SubscribeStar serves every upload as a
+direct file via `/post_uploads?payload=...` (plain GET — no Mux/HLS, no yt-dlp),
+and the full post body is already present in the feed HTML (no detail-endpoint
+enrichment). PURE: no DB; the seen-skip is an injected predicate.
+
+On-disk layout (matches gallery-dl's subscribestar config
+`{date:%Y-%m-%d}_{id}_{title[:40]}` / `{num:>02}_{filename}`): SubscribeStar posts
+have no title, so the directory is `<YYYY-MM-DD>_<post_id>_` — the display title is
+synthesized by the importer from the body, so the bare dir name is on-disk only.
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import requests
+
+from .native_ingest_common import (
+    BaseNativeDownloader,
+    MediaOutcome,
+    PostRecordOutcome,
+    post_dir_name,
+    sanitize_segment,
+)
+
+log = logging.getLogger(__name__)
+
+
+class SubscribeStarDownloader(BaseNativeDownloader):
+    """Download resolved SubscribeStar media to gallery-dl's on-disk layout.
+    Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry +
+    Range-resume) and validation/quarantine. No video branch (SubscribeStar serves
+    files directly via /post_uploads) and no detail-fetch (the body is already in
+    the feed HTML). PURE: no DB."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        super().__init__(
+            images_root, cookies_path, platform="subscribestar",
+            validate=validate, rate_limit=rate_limit, session=session,
+        )
+
+    # -- public ------------------------------------------------------------
+
+    def download_post(
+        self,
+        post: dict,
+        media_items: list,
+        artist_slug: str,
+        *,
+        is_seen: Callable[[object], bool] = lambda m: False,
+        should_stop: Callable[[], bool] = lambda: False,
+        recapture: bool = False,
+    ) -> list[MediaOutcome]:
+        """Download every media item of one post; return per-item outcomes.
+        Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
+        recapture surfacing) minus the video branch."""
+        post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
+        outcomes: list[MediaOutcome] = []
+        for i, media in enumerate(media_items, start=1):
+            if should_stop():
+                break
+            try:
+                outcomes.append(
+                    self._download_one(
+                        post, media, post_dir, artist_slug, i, is_seen,
+                        recapture=recapture,
+                    )
+                )
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "SubscribeStar media failed (post %s, item %d): %s",
+                    post.get("id"), i, exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    # -- per-item ----------------------------------------------------------
+
+    def _download_one(
+        self,
+        post: dict,
+        media,
+        post_dir: Path,
+        artist_slug: str,
+        index: int,
+        is_seen: Callable[[object], bool],
+        *,
+        recapture: bool = False,
+    ) -> MediaOutcome:
+        seen = is_seen(media)
+        if seen and not recapture:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        nn = f"{index:02d}"
+        media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}")
+
+        if media_path.exists():  # tier-2: already on disk
+            return MediaOutcome(
+                media=media, status="skipped_disk", path=media_path, error=None
+            )
+        # recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
+        if seen:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        post_dir.mkdir(parents=True, exist_ok=True)
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+
+        out_path = self._fetch_get(media.url, media_path)
+        reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
+        if reason is not None:
+            return MediaOutcome(
+                media=media, status="quarantined", path=quarantine_dest, error=reason,
+            )
+        self._write_sidecar(post, out_path, source_url=media.url)
+        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
+
+    # The plain-GET streaming path (_fetch_get / _fetch_to_file) and
+    # _validate_path are inherited from BaseNativeDownloader.
+
+    # -- sidecar -----------------------------------------------------------
+
+    def _write_sidecar(
+        self, post: dict, media_path: Path, *, source_url: str | None = None
+    ) -> Path:
+        """Per-media sidecar — post-first (#856): image identity ONLY
+        (category/id/source_url). The post body/links live solely in _post.json."""
+        return self._write_sidecar_data(
+            post, media_path.with_suffix(".json"), source_url=source_url, minimal=True,
+        )
+
+    def _write_sidecar_data(
+        self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
+        minimal: bool = False,
+    ) -> Path:
+        """Serialize the post's metadata. minimal=True → per-media sidecar (image
+        identity only); else the full post record (body/title/date/url)."""
+        if minimal:
+            data = {"category": "subscribestar", "id": str(post.get("id") or "")}
+            if source_url:
+                data["source_url"] = source_url
+            sidecar_path.write_text(json.dumps(data, indent=2))
+            return sidecar_path
+        attrs = post.get("attributes") or {}
+        content = attrs.get("content")
+        # SubscribeStar synthesizes the post permalink from the id (matches the
+        # platforms/subscribestar.py derive_post_url helper).
+        pid = str(post.get("id") or "")
+        data = {
+            "category": "subscribestar",
+            "id": pid,
+            "post_id": pid,  # ensures derive_post_url has its key on the native path
+            "title": attrs.get("title") if isinstance(attrs.get("title"), str) else "",
+            "content": content if isinstance(content, str) else "",
+            "published_at": attrs.get("published_at"),
+        }
+        if source_url:
+            data["source_url"] = source_url
+        sidecar_path.write_text(json.dumps(data, indent=2))
+        return sidecar_path
+
+    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
+        """Write the post-first `_post.json` (body/links/metadata) — the sole
+        writer of the post record on the native path. SubscribeStar's body is
+        already in the feed HTML, so no detail-fetch is needed."""
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
+        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
+        pid = str(post.get("id") or "")
+        if not pid:
+            return PostRecordOutcome(
+                path=None, post_type=post_type, title=title, body_chars=0,
+            )
+        post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
+        post_dir.mkdir(parents=True, exist_ok=True)
+        path = self._write_sidecar_data(post, post_dir / "_post.json")
+        body = attrs.get("content")
+        body_chars = len(body) if isinstance(body, str) else 0
+        return PostRecordOutcome(
+            path=path, post_type=post_type, title=title, body_chars=body_chars,
+        )

backend/app/services/subscribestar_ingester.py

@@ -0,0 +1,109 @@
+"""Native SubscribeStar ingester — the SubscribeStar ADAPTER over the
+platform-agnostic core (`ingest_core.Ingester`).
+
+Thin counterpart to patreon_ingester: wires the SubscribeStar client/downloader/
+ledger models/constraints/key into the core and supplies the SubscribeStar
+failure mapping. The three modes (tick / backfill / recovery / recapture), the
+seen + dead-letter ledgers, cursor checkpointing, and the post-first capture all
+live in the core — identical to Patreon. `download_service.download_source`
+drives `SubscribeStarIngester.run` exactly as it drives the Patreon one.
+
+`campaign_id` is the creator URL (the client derives host + slug from it), so no
+campaign-id resolver is needed. FC runs on a plain-HTTP homelab; nothing here
+uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from ..models import SubscribeStarFailedMedia, SubscribeStarSeenMedia
+from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
+from .subscribestar_client import MediaItem, SubscribeStarAPIError, SubscribeStarClient
+from .subscribestar_downloader import SubscribeStarDownloader
+
+__all__ = [
+    "DEAD_LETTER_THRESHOLD",
+    "SubscribeStarIngester",
+    "_ledger_key",
+    "verify_subscribestar_credential",
+]
+
+log = logging.getLogger(__name__)
+
+_LEDGER_KEY_MAX = 128
+
+
+def _ledger_key(media: MediaItem) -> str:
+    """Stable per-media identity for the cross-run seen-ledger. SubscribeStar's
+    full-res URL is an opaque `/post_uploads?payload=...` (no content hash), so
+    `media.filehash` is normally None and the stable proxy is the gallery item id
+    scoped to its post: `<post_id>:<media_id>`. Bounded to the column width."""
+    if media.filehash:
+        return media.filehash
+    return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
+
+
+class SubscribeStarIngester(Ingester):
+    """Walk a SubscribeStar creator's posts, download unseen media, return a
+    `DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
+    `downloader` are injectable seams so unit tests run without network."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None,
+        session_factory: Callable[[], object],
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        request_sleep: float = 0.0,
+        client: SubscribeStarClient | None = None,
+        downloader: SubscribeStarDownloader | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        resolved_client = (
+            client
+            if client is not None
+            else SubscribeStarClient(cookies_path, request_sleep=request_sleep)
+        )
+        resolved_downloader = (
+            downloader
+            if downloader is not None
+            else SubscribeStarDownloader(
+                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
+            )
+        )
+        super().__init__(
+            client=resolved_client,
+            downloader=resolved_downloader,
+            session_factory=session_factory,
+            seen_model=SubscribeStarSeenMedia,
+            failed_model=SubscribeStarFailedMedia,
+            seen_constraint="uq_subscribestar_seen_media_source_id",
+            failed_constraint="uq_subscribestar_failed_media_source_id",
+            ledger_key=_ledger_key,
+            platform="subscribestar",
+            error_base=SubscribeStarAPIError,
+            # API_DRIFT message phrasing; the base Ingester._failure_result owns
+            # the auth/drift/HTTP→error_type mapping (shared across platforms).
+            drift_label="SubscribeStar markup",
+        )
+
+
+async def verify_subscribestar_credential(
+    url: str,
+    cookies_path: str | None,
+    overrides: dict | None,
+) -> tuple[bool | None, str]:
+    """Native SubscribeStar credential probe — fetches ONE feed page via
+    SubscribeStarClient.verify_auth. `campaign_id` is just the creator URL (no
+    resolver). Returns the uniform `(ok, message)` contract so
+    download_backends.verify_credential treats it like the gallery-dl probe."""
+    client = SubscribeStarClient(cookies_path)
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, client.verify_auth, url)

backend/app/services/tag_directory_service.py

@@ -53,12 +53,28 @@ class TagDirectoryService:
             raise ValueError("limit must be between 1 and 200")
 
         fandom = aliased(Tag)
-        count_col = func.count(image_tag.c.image_record_id).label("image_count")
-        stmt = (
-            select(Tag, fandom.name.label("fandom_name"), count_col)
-            .outerjoin(fandom, Tag.fandom_id == fandom.id)
-            .outerjoin(image_tag, image_tag.c.tag_id == Tag.id)
-            .group_by(Tag.id, fandom.name)
+        member = aliased(Tag)
+        # image_count aggregates the tag's images INCLUDING, for a fandom, every
+        # image carrying one of its characters. `member` is the tag actually on
+        # the image; an image counts for the outer Tag if that applied tag IS the
+        # Tag (direct) OR is a character whose fandom_id is the Tag (the fandom
+        # leg). Both legs correlate the OUTER Tag.id at a SINGLE level — a nested
+        # `tag_id IN (SELECT ... WHERE member.fandom_id == Tag.id)` does NOT
+        # correlate and silently counts every fandom-character globally (~every
+        # tag collapses to the same inflated number). DISTINCT so an image with
+        # the fandom AND ≥1 of its characters counts once; a correlated scalar
+        # subquery keeps it 1 row per tag with no group_by.
+        count_col = (
+            select(func.count(image_tag.c.image_record_id.distinct()))
+            .select_from(image_tag)
+            .join(member, member.id == image_tag.c.tag_id)
+            .where(or_(image_tag.c.tag_id == Tag.id, member.fandom_id == Tag.id))
+            .correlate(Tag)
+            .scalar_subquery()
+            .label("image_count")
+        )
+        stmt = select(Tag, fandom.name.label("fandom_name"), count_col).outerjoin(
+            fandom, Tag.fandom_id == fandom.id
         )
         if kind is not None:
             stmt = stmt.where(Tag.kind == kind)
@@ -101,19 +117,34 @@ class TagDirectoryService:
     async def _previews(self, tag_ids: list[int]) -> dict[int, list[str]]:
         if not tag_ids:
             return {}
-        rn = func.row_number().over(
-            partition_by=image_tag.c.tag_id,
-            order_by=image_tag.c.image_record_id.desc(),
-        ).label("rn")
-        sub = (
+        # Preview pool per card = images tagged with the card's tag DIRECTLY,
+        # UNION images carrying a character whose fandom_id is the card's tag (so
+        # a fandom card previews its characters' images). UNION dedups the
+        # overlap. For non-fandom cards the via-fandom leg is empty.
+        member = aliased(Tag)
+        direct = select(
+            image_tag.c.tag_id.label("card_id"),
+            image_tag.c.image_record_id.label("image_record_id"),
+        ).where(image_tag.c.tag_id.in_(tag_ids))
+        via_fandom = (
             select(
-                image_tag.c.tag_id.label("tag_id"),
+                member.fandom_id.label("card_id"),
                 image_tag.c.image_record_id.label("image_record_id"),
-                rn,
             )
-            .where(image_tag.c.tag_id.in_(tag_ids))
-            .subquery()
+            .select_from(image_tag)
+            .join(member, member.id == image_tag.c.tag_id)
+            .where(member.fandom_id.in_(tag_ids))
         )
+        scoped = direct.union(via_fandom).subquery()
+        rn = func.row_number().over(
+            partition_by=scoped.c.card_id,
+            order_by=scoped.c.image_record_id.desc(),
+        ).label("rn")
+        sub = select(
+            scoped.c.card_id.label("tag_id"),
+            scoped.c.image_record_id.label("image_record_id"),
+            rn,
+        ).subquery()
         stmt = (
             select(
                 sub.c.tag_id,

backend/app/services/tag_query.py

@@ -0,0 +1,90 @@
+"""Shared tag-with-fandom query columns, serialization, and membership.
+
+`image_in_tag_scope`/`image_in_any_tag_scope` are the SINGLE source of truth for
+"which images belong to a tag" — a fandom tag also owns the images of its
+characters (Tag.fandom_id), derived at query time rather than materialized. The
+gallery scope, directory count/previews, and the cleanup impact count all route
+through them so a fandom can never under-count its characters' images.
+
+Resolving a character tag's fandom NAME via a Tag self-join (Tag.fandom_id -> the
+fandom Tag) and serializing the canonical
+``{id, name, kind, fandom_id, fandom_name}`` dict were hand-written in
+TagService.autocomplete / .list_for_image and GalleryService.get_image_with_tags
+(the last two added by the fandom-on-chip feature, 978f49a). One source now
+(DRY pattern sweep 2026-06-10) so a new tag field is added in exactly one place.
+
+TagDirectoryService selects the FULL Tag ORM plus an image-count aggregate (a
+different select shape), so it keeps its own variant — not folded in here.
+"""
+
+from sqlalchemy import exists, or_, select
+from sqlalchemy.orm import aliased
+
+from ..models import ImageRecord, Tag
+from ..models.tag import image_tag
+
+
+def _fandom_member_char_ids(tids):
+    """Subquery of character tag ids owned by any fandom in `tids` (via
+    Tag.fandom_id). Empty for non-fandom tids, so callers can OR it in
+    unconditionally — a fandom aggregates its characters' images, every other
+    kind degrades to direct-only."""
+    char = aliased(Tag)
+    return select(char.id).where(char.fandom_id.in_(tids))
+
+
+def image_in_tag_scope(tid):
+    """Correlated EXISTS: the current ImageRecord belongs to tag `tid` — either
+    tagged with it DIRECTLY, or (when `tid` is a fandom) carrying a character
+    whose fandom_id == `tid`. This is the SINGLE membership predicate shared by
+    the gallery scope, the directory count/previews, and the cleanup impact
+    count, so a fandom never under-counts the images of its characters."""
+    return exists().where(
+        image_tag.c.image_record_id == ImageRecord.id,
+        or_(
+            image_tag.c.tag_id == tid,
+            image_tag.c.tag_id.in_(_fandom_member_char_ids([tid])),
+        ),
+    )
+
+
+def image_in_any_tag_scope(tids):
+    """Like `image_in_tag_scope` but for an OR-group: the image carries AT LEAST
+    ONE of `tids` directly, or a character of any fandom in `tids`."""
+    return exists().where(
+        image_tag.c.image_record_id == ImageRecord.id,
+        or_(
+            image_tag.c.tag_id.in_(tids),
+            image_tag.c.tag_id.in_(_fandom_member_char_ids(tids)),
+        ),
+    )
+
+
+def fandom_join_alias():
+    """A Tag self-join alias for resolving a tag's fandom name. Outerjoin it on
+    ``Tag.fandom_id == <alias>.c.id`` and select via `tag_columns(alias)`."""
+    return Tag.__table__.alias("fandom_lookup")
+
+
+def tag_columns(fandom_alias):
+    """The canonical (id, name, kind, fandom_id, fandom_name) column set for a
+    tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`)."""
+    return [
+        Tag.id,
+        Tag.name,
+        Tag.kind,
+        Tag.fandom_id,
+        fandom_alias.c.name.label("fandom_name"),
+    ]
+
+
+def serialize_tag(row) -> dict:
+    """Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the
+    canonical tag dict. `kind` may be a TagKind enum or a plain string."""
+    return {
+        "id": row.id,
+        "name": row.name,
+        "kind": row.kind.value if hasattr(row.kind, "value") else row.kind,
+        "fandom_id": row.fandom_id,
+        "fandom_name": row.fandom_name,
+    }

backend/app/services/tag_service.py

@@ -1,16 +1,38 @@
 """Tag CRUD + autocomplete + image-tag association."""
 
+import logging
+import time
 from collections.abc import Sequence
 from dataclasses import dataclass
 
 from sqlalchemy import and_, case, exists, func, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..models import Tag, TagKind, image_tag
 from ..models.tag_allowlist import TagAllowlist
 from ..models.tag_reference_embedding import TagReferenceEmbedding
+from .db_helpers import get_or_create
+from .tag_query import fandom_join_alias, tag_columns
+
+log = logging.getLogger(__name__)
+
+
+def normalize_tag_name(name: str) -> str:
+    """Canonical tag form (#701): collapse whitespace + capitalize the first
+    letter of each word, PRESERVING the rest of the word.
+
+    Per-word capitalize (NOT str.title(), which mangles apostrophes:
+    don't → Don'T). The word tail is left untouched so acronyms survive
+    (DC stays DC, NSFW stays NSFW) — operator-revised 2026-06-06: protecting
+    acronyms matters more than folding ALL-CAPS input. This MATCHES
+    ml/tag_name._title_word, so a tag suggested by the Camie tagger keeps the
+    exact casing the suggestion UI showed when it round-trips through the
+    create endpoint on Accept.
+    """
+    return " ".join(
+        w[:1].upper() + w[1:] for w in (name or "").split()
+    )
 
 
 class TagValidationError(ValueError):
@@ -57,6 +79,23 @@ class MergeResult:
     source_deleted: bool
 
 
+@dataclass(frozen=True)
+class MergePreview:
+    """Non-mutating projection of what merge(source→target) would do (#8).
+    Shares the apply's predicates (rule 93) so the preview can't drift."""
+    source_id: int
+    source_name: str
+    target_id: int
+    target_name: str
+    compatible: bool                # same kind + fandom — would apply succeed?
+    images_moving: int              # source links that move (image lacks target)
+    images_already_on_target: int   # source links dropped (image has target)
+    source_total: int               # images carrying source
+    series_pages: int               # series pages repointed
+    will_alias: bool                # source name kept as a protective alias
+    sample_thumbnails: list[str]    # a few of the moving images
+
+
 class TagService:
     def __init__(self, session: AsyncSession):
         self.session = session
@@ -71,6 +110,10 @@ class TagService:
         - if fandom_id is set, the referenced tag must exist and have
           kind == TagKind.fandom
         """
+        # NOTE: case is NOT normalized here — find_or_create is the shared path
+        # the ML tagger / allowlist also use, and Title-Casing the booru
+        # vocabulary breaks allowlist matching. Display-casing (Title Case) is
+        # applied at the user-entry layer (api/tags create_tag) only (#701).
         name = name.strip()
         if not name:
             raise TagValidationError("Tag name cannot be empty")
@@ -93,28 +136,26 @@ class TagService:
         # inserts; without the savepoint the outer transaction would
         # poison and the calling request crashes. Mirrors
         # importer._get_or_create.
+        # Case-insensitive match (#701): a normalized (Title Case) input must
+        # find an existing differently-cased tag instead of forking a duplicate.
         stmt = (
             select(Tag)
-            .where(Tag.name == name)
+            .where(func.lower(Tag.name) == name.lower())
             .where(Tag.kind == kind)
             .where(
                 Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id
             )
+            .order_by(Tag.id)
+            .limit(1)
         )
-        existing = (await self.session.execute(stmt)).scalar_one_or_none()
-        if existing:
-            return existing
-
-        sp = await self.session.begin_nested()
-        try:
+        async def _create() -> Tag:
             new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id)
             self.session.add(new_tag)
             await self.session.flush()
-            await sp.commit()
             return new_tag
-        except IntegrityError:
-            await sp.rollback()
-            return (await self.session.execute(stmt)).scalar_one()
+
+        tag, _created = await get_or_create(self.session, stmt, _create)
+        return tag
 
     async def autocomplete(
         self,
@@ -136,7 +177,7 @@ class TagService:
             else_=2,
         )
 
-        fandom_alias = Tag.__table__.alias("fandom_lookup")
+        fandom_alias = fandom_join_alias()
         image_count = (
             select(func.count(image_tag.c.image_record_id))
             .where(image_tag.c.tag_id == Tag.id)
@@ -146,11 +187,7 @@ class TagService:
 
         stmt = (
             select(
-                Tag.id,
-                Tag.name,
-                Tag.kind,
-                Tag.fandom_id,
-                fandom_alias.c.name.label("fandom_name"),
+                *tag_columns(fandom_alias),
                 image_count.label("image_count"),
             )
             .select_from(Tag.__table__.outerjoin(
@@ -196,14 +233,23 @@ class TagService:
             )
         )
 
-    async def list_for_image(self, image_id: int) -> Sequence[Tag]:
+    async def list_for_image(self, image_id: int) -> Sequence:
+        """Tags on an image, ordered (kind, name). Each row carries the fandom's
+        NAME (not just fandom_id) via a self-join on Tag, so the UI can label a
+        character chip with its fandom without an N+1 (mirrors the
+        autocomplete/directory resolution)."""
+        fandom_alias = fandom_join_alias()
         stmt = (
-            select(Tag)
-            .join(image_tag, image_tag.c.tag_id == Tag.id)
+            select(*tag_columns(fandom_alias))
+            .select_from(
+                Tag.__table__
+                .join(image_tag, image_tag.c.tag_id == Tag.id)
+                .outerjoin(fandom_alias, Tag.fandom_id == fandom_alias.c.id)
+            )
             .where(image_tag.c.image_record_id == image_id)
             .order_by(Tag.kind.asc(), Tag.name.asc())
         )
-        return (await self.session.execute(stmt)).scalars().all()
+        return (await self.session.execute(stmt)).all()
 
     async def _keep_as_alias(self, tag_id: int) -> bool:
         """A merged-away tag's old name must survive as an alias iff the ML
@@ -249,9 +295,11 @@ class TagService:
         if tag is None:
             raise TagValidationError(f"Tag {tag_id} not found")
 
+        # Case-insensitive clash (#701) — renaming onto a differently-cased tag
+        # is still a merge, not a silent fork.
         clash_stmt = (
             select(Tag)
-            .where(Tag.name == new_name)
+            .where(func.lower(Tag.name) == new_name.lower())
             .where(Tag.kind == tag.kind)
             .where(
                 Tag.fandom_id.is_(None)
@@ -259,6 +307,8 @@ class TagService:
                 else Tag.fandom_id == tag.fandom_id
             )
             .where(Tag.id != tag_id)
+            .order_by(Tag.id)
+            .limit(1)
         )
         clash = (await self.session.execute(clash_stmt)).scalar_one_or_none()
         if clash is not None:
@@ -343,6 +393,96 @@ class TagService:
         await self.session.flush()
         return tag
 
+    async def merge_preview(
+        self, source_id: int, target_id: int
+    ) -> MergePreview:
+        """Read-only dry-run of merge(source→target): the same counts the apply
+        would produce, plus a thumbnail sample of the images that move. Raises
+        TagValidationError only for missing/self-merge (so the UI can render a
+        kind/fandom-mismatch warning rather than a hard error)."""
+        if source_id == target_id:
+            raise TagValidationError("Cannot merge a tag into itself")
+        source = await self.session.get(Tag, source_id)
+        target = await self.session.get(Tag, target_id)
+        if source is None or target is None:
+            raise TagValidationError("Tag not found")
+
+        compatible = source.kind == target.kind and (
+            (source.fandom_id or 0) == (target.fandom_id or 0)
+        )
+
+        # Mirrors _repoint_image_tags: links whose image already has target are
+        # dropped (already_on_target); the rest move.
+        target_images = select(image_tag.c.image_record_id).where(
+            image_tag.c.tag_id == target_id
+        )
+        moving = await self.session.scalar(
+            select(func.count())
+            .select_from(image_tag)
+            .where(
+                image_tag.c.tag_id == source_id,
+                image_tag.c.image_record_id.notin_(target_images),
+            )
+        )
+        already = await self.session.scalar(
+            select(func.count())
+            .select_from(image_tag)
+            .where(
+                image_tag.c.tag_id == source_id,
+                image_tag.c.image_record_id.in_(target_images),
+            )
+        )
+
+        from ..models.series_page import SeriesPage
+
+        series_pages = await self.session.scalar(
+            select(func.count())
+            .select_from(SeriesPage)
+            .where(SeriesPage.series_tag_id == source_id)
+        )
+
+        will_alias = await self._keep_as_alias(source_id)
+        sample = await self._merge_sample_thumbnails(source_id, target_images)
+
+        return MergePreview(
+            source_id=source_id,
+            source_name=source.name,
+            target_id=target_id,
+            target_name=target.name,
+            compatible=compatible,
+            images_moving=moving or 0,
+            images_already_on_target=already or 0,
+            source_total=(moving or 0) + (already or 0),
+            series_pages=series_pages or 0,
+            will_alias=will_alias,
+            sample_thumbnails=sample,
+        )
+
+    async def _merge_sample_thumbnails(
+        self, source_id: int, target_images
+    ) -> list[str]:
+        """Up to 6 thumbnails of the images that WOULD move (carry source, not
+        target) — a sanity check that the merge targets the right content."""
+        from ..models import ImageRecord
+        from .gallery_service import thumbnail_url
+
+        rows = (
+            await self.session.execute(
+                select(
+                    ImageRecord.thumbnail_path,
+                    ImageRecord.sha256,
+                    ImageRecord.mime,
+                )
+                .join(image_tag, image_tag.c.image_record_id == ImageRecord.id)
+                .where(
+                    image_tag.c.tag_id == source_id,
+                    image_tag.c.image_record_id.notin_(target_images),
+                )
+                .limit(6)
+            )
+        ).all()
+        return [thumbnail_url(tp, sha, mime) for tp, sha, mime in rows]
+
     async def merge(self, source_id: int, target_id: int) -> MergeResult:
         """Transactionally repoint every FK from source→target, optionally
         keep source's name as a tagger alias, delete source. Atomic: any
@@ -482,14 +622,33 @@ class TagService:
         )
 
     async def _repoint_series_pages(self, src: int, tgt: int) -> None:
+        from ..models.series_chapter import SeriesChapter
         from ..models.series_page import SeriesPage
 
-        # image_id is UNIQUE across series_page and src != tgt, so an
-        # image in src's series cannot already be in tgt's — no collision.
+        # Move src's dividers onto tgt. Their anchor_page_id stays valid — the
+        # anchored pages move too — and anchor is globally UNIQUE so no collision.
+        await self.session.execute(
+            update(SeriesChapter)
+            .where(SeriesChapter.series_tag_id == src)
+            .values(series_tag_id=tgt)
+        )
+        # Append src's pages after tgt's so the merged series stays ONE clean
+        # series-global run (image_id is UNIQUE, so the two page sets are
+        # disjoint). Offset src's page_number past tgt's current max.
+        tgt_max = (
+            await self.session.scalar(
+                select(
+                    func.coalesce(func.max(SeriesPage.page_number), 0)
+                ).where(SeriesPage.series_tag_id == tgt)
+            )
+        ) or 0
         await self.session.execute(
             update(SeriesPage)
             .where(SeriesPage.series_tag_id == src)
-            .values(series_tag_id=tgt)
+            .values(
+                series_tag_id=tgt,
+                page_number=SeriesPage.page_number + tgt_max,
+            )
         )
 
     async def _repoint_fandom_children(
@@ -501,50 +660,248 @@ class TagService:
             update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
         )
 
+    async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]:
+        """image_tag row counts keyed by tag_id, for the survivor heuristic
+        (the best-connected tag in a collision group survives → fewest
+        FK repoints). Tags with zero associations are absent from the map."""
+        if not tag_ids:
+            return {}
+        rows = (
+            await self.session.execute(
+                select(image_tag.c.tag_id, func.count())
+                .where(image_tag.c.tag_id.in_(tag_ids))
+                .group_by(image_tag.c.tag_id)
+            )
+        ).all()
+        return {tid: int(n) for tid, n in rows}
+
     async def _create_protective_aliases(
         self, src_name: str, src_kind: TagKind, tgt: int
     ) -> bool:
-        """One alias per category the tagger has actually emitted for
-        src_name (so future predictions resolve to target); fall back to
-        the tag's kind when the tagger never predicted this exact name.
-        Idempotent — never clobbers a pre-existing operator alias."""
+        """Alias (src_name, category) -> tgt so future tagger predictions of
+        src_name resolve to the merge survivor. Idempotent — never clobbers a
+        pre-existing operator alias.
+
+        The category is the tag's kind. The tagger's tag_to_category map is
+        one-to-one (a name has exactly ONE category), and a tag's kind is set
+        from that category when it's created — so kind already IS the tagger's
+        category for this name. This used to SELECT DISTINCT category by scanning
+        every image_record's tagger_predictions JSON (an unindexed full scan that,
+        on a large library, held a series_page lock for minutes inside the merge
+        and blocked migration 0040 — operator-flagged 2026-06-07). That scan only
+        ever rediscovered the kind, so it's gone."""
         from ..models.tag_alias import TagAlias
 
-        rows = (
-            await self.session.execute(
-                text(
-                    "SELECT DISTINCT "
-                    "  (tagger_predictions::jsonb -> :n ->> 'category') "
-                    "  AS cat "
-                    "FROM image_record "
-                    "WHERE tagger_predictions IS NOT NULL "
-                    "  AND (tagger_predictions::jsonb) ? :n"
-                ),
-                {"n": src_name},
+        category = src_kind.value if hasattr(src_kind, "value") else str(src_kind)
+        res = await self.session.execute(
+            pg_insert(TagAlias)
+            .values(
+                alias_string=src_name,
+                alias_category=category,
+                canonical_tag_id=tgt,
             )
-        ).all()
-        categories = {r.cat for r in rows if r.cat}
-        if not categories:
-            kind_val = (
-                src_kind.value
-                if hasattr(src_kind, "value")
-                else str(src_kind)
+            .on_conflict_do_nothing(
+                index_elements=["alias_string", "alias_category"]
             )
-            categories = {kind_val}
+        )
+        return bool(res.rowcount)
 
-        created = False
-        for cat in categories:
-            res = await self.session.execute(
-                pg_insert(TagAlias)
-                .values(
-                    alias_string=src_name,
-                    alias_category=cat,
-                    canonical_tag_id=tgt,
-                )
-                .on_conflict_do_nothing(
-                    index_elements=["alias_string", "alias_category"]
-                )
+
+# ---------------------------------------------------------------------------
+# #714: retro-normalize existing tags to the #701 canonical (Title Case +
+# collapsed whitespace) and merge case/whitespace-variant duplicates.
+# ---------------------------------------------------------------------------
+
+_NORMALIZE_SAMPLE_CAP = 50
+
+
+def _group_existing_tags(
+    rows,
+) -> dict[tuple, list[tuple[int, str]]]:
+    """Group (id, name, kind, fandom_id) rows by their post-normalization
+    identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that
+    would collapse to the same canonical lives in ONE group, so the canonical
+    form is unique within (kind, fandom) once each group is resolved."""
+    groups: dict[tuple, list[tuple[int, str]]] = {}
+    for tag_id, name, kind, fandom_id in rows:
+        canonical = normalize_tag_name(name)
+        key = (kind, fandom_id if fandom_id is not None else -1, canonical)
+        groups.setdefault(key, []).append((tag_id, name))
+    return groups
+
+
+def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool:
+    """A group is already canonical iff it's a single member whose name equals
+    the canonical form. Anything else (a collision, or a lone mis-cased tag)
+    needs work."""
+    if len(members) > 1:
+        return True
+    return members[0][1] != canonical
+
+
+def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
+    """The tag with the most image associations (→ fewest FK repoints when it
+    survives), tie-broken to the lowest id for determinism. Module-level so the
+    key closes over its parameter, not a loop variable (ruff B023)."""
+    return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid))
+
+
+async def normalize_existing_tags(
+    session: AsyncSession,
+    *,
+    dry_run: bool = False,
+    time_budget_seconds: float | None = None,
+) -> dict:
+    """Convert the back-catalog to the #701 canonical tag form.
+
+    For each (kind, fandom, canonical) group: pick a survivor, merge any
+    case/whitespace-variant siblings INTO it via the tested merge path
+    (TagService._do_merge — FK repoints + protective aliases), then rename the
+    survivor to the canonical form. Idempotent: a group that is already a lone
+    canonical tag is a no-op, so re-running is safe.
+
+    A first run over a fresh back-catalog can touch tens of thousands of tags
+    (the whole booru-derived vocabulary needs recasing) and won't finish inside
+    one Celery time limit — it timed out at 40 min (operator-flagged 2026-06-07).
+    `time_budget_seconds` time-boxes the live run: it stops cleanly at the budget
+    and reports `partial`/`remaining` so the caller can re-enqueue and continue.
+    Because it commits per group and is idempotent, the next run just picks up
+    the groups still needing change.
+
+    dry_run=True returns a projection (counts + a sample of the changes) with no
+    mutations. Live runs commit per group and isolate failures per group so one
+    bad group can't strand the rest.
+
+    Returns (dry_run):
+      {"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R,
+       "total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
+    Returns (live):
+      {"groups_processed", "merged", "renamed", "aliases_created", "errors",
+       "total_changes", "remaining", "partial", "sample": [...]}
+    """
+    rows = (
+        await session.execute(
+            select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
+        )
+    ).all()
+    groups = _group_existing_tags(rows)
+
+    # Deterministic sample/ordering: by kind then canonical name.
+    touched = sorted(
+        (
+            (key, members)
+            for key, members in groups.items()
+            if _group_needs_change(key[2], members)
+        ),
+        key=lambda km: (
+            km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]),
+            km[0][2].lower(),
+        ),
+    )
+
+    sample = [
+        {
+            "to": key[2],
+            "from": [name for _id, name in members],
+            "kind": key[0].value if hasattr(key[0], "value") else str(key[0]),
+            "merge": len(members) > 1,
+        }
+        for key, members in touched[:_NORMALIZE_SAMPLE_CAP]
+    ]
+
+    if dry_run:
+        collisions = sum(1 for key, m in touched if len(m) > 1)
+        tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1)
+        # A group renames iff the canonical form isn't already one of its
+        # members' exact names (else that member is picked as survivor → no
+        # rename, the rest merge into it).
+        tags_to_rename = sum(
+            1
+            for key, m in touched
+            if key[2] not in {name for _id, name in m}
+        )
+        return {
+            "groups": len(groups),
+            "collisions": collisions,
+            "tags_to_merge": tags_to_merge,
+            "tags_to_rename": tags_to_rename,
+            "total_changes": len(touched),
+            "sample": sample,
+        }
+
+    svc = TagService(session)
+    summary = {
+        "groups_processed": 0,
+        "merged": 0,
+        "renamed": 0,
+        "aliases_created": 0,
+        "errors": 0,
+        "total_changes": len(touched),
+        "remaining": len(touched),
+        "partial": False,
+        "sample": sample,
+    }
+    start = time.monotonic()
+    log.info(
+        "normalize_existing_tags: %d group(s) need changes (budget=%ss)",
+        len(touched), time_budget_seconds,
+    )
+    for done, (key, members) in enumerate(touched):
+        # Time-box: stop cleanly before the Celery limit kills us mid-group and
+        # strands the run as a timeout. The caller re-enqueues to finish the
+        # rest (idempotent — already-canonical groups are skipped next pass).
+        if (
+            time_budget_seconds is not None
+            and time.monotonic() - start >= time_budget_seconds
+        ):
+            summary["partial"] = True
+            summary["remaining"] = len(touched) - done
+            break
+        # Heartbeat so a long run is diagnosable instead of silent — the timeout
+        # operator-flagged 2026-06-07 produced zero logs because the only log was
+        # per finished group and it was stuck mid-group on a lock.
+        if done and done % 25 == 0:
+            log.info(
+                "normalize_existing_tags: %d/%d groups (%d merged, %d errors, "
+                "%.0fs elapsed)",
+                done, len(touched), summary["merged"], summary["errors"],
+                time.monotonic() - start,
             )
-            if res.rowcount:
-                created = True
-        return created
+        canonical = key[2]
+        names_by_id = dict(members)
+        # Survivor: prefer a member already named canonically (no rename, no
+        # self-alias); else the best-connected (fewest FK repoints); else
+        # lowest id for determinism.
+        survivor_id = next(
+            (tid for tid, name in members if name == canonical), None
+        )
+        if survivor_id is None:
+            counts = await svc._image_assoc_counts(list(names_by_id))
+            survivor_id = _best_connected(list(names_by_id), counts)
+        loser_ids = [tid for tid in names_by_id if tid != survivor_id]
+        try:
+            survivor = await session.get(Tag, survivor_id)
+            for loser_id in loser_ids:
+                loser = await session.get(Tag, loser_id)
+                if loser is None:
+                    continue
+                result = await svc._do_merge(loser, survivor)
+                if result.alias_created:
+                    summary["aliases_created"] += 1
+            if survivor.name != canonical:
+                survivor.name = canonical
+                await session.flush()
+                summary["renamed"] += 1
+            await session.commit()
+            summary["groups_processed"] += 1
+            summary["merged"] += len(loser_ids)
+        except Exception as exc:  # one bad group must not strand the rest
+            await session.rollback()
+            summary["errors"] += 1
+            log.warning(
+                "tag normalize failed for group %r: %s", canonical, exc
+            )
+    else:
+        # Loop finished without hitting the time budget — nothing left to do.
+        summary["remaining"] = 0
+    return summary

backend/app/tasks/_async_session.py

@@ -15,8 +15,15 @@ from sqlalchemy.pool import NullPool
 from ..config import get_config
 
 
-def async_session_factory():
-    """Return ``(sessionmaker, engine)`` bound to a fresh async engine."""
+def async_session_factory(*, server_settings: dict | None = None):
+    """Return ``(sessionmaker, engine)`` bound to a fresh async engine.
+
+    ``server_settings`` (optional) are applied by asyncpg as per-connection GUCs
+    on connect. Because NullPool opens a fresh real connection per checkout, every
+    transaction in the task inherits them — used to set ``lock_timeout`` so a
+    statement blocked on a lock fails fast instead of hanging to the Celery hard
+    limit (operator-flagged normalize_tags timeout 2026-06-07).
+    """
     cfg = get_config()
     # NullPool: this engine lives for ONE task (created + disposed per
     # asyncio.run loop), so intra-task connection pooling buys nothing and
@@ -26,5 +33,13 @@ def async_session_factory():
     # phase 3 (asyncpg ConnectionDoesNotExistError, Anduo #40014). NullPool
     # opens a fresh real connection on each checkout, so phase 3 always
     # reconnects clean; pre_ping is then redundant.
-    engine = create_async_engine(cfg.database_url, future=True, poolclass=NullPool)
+    connect_args = {}
+    if server_settings:
+        connect_args["server_settings"] = {
+            k: str(v) for k, v in server_settings.items()
+        }
+    engine = create_async_engine(
+        cfg.database_url, future=True, poolclass=NullPool,
+        connect_args=connect_args,
+    )
     return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine

backend/app/tasks/admin.py

@@ -14,9 +14,11 @@ from __future__ import annotations
 import logging
 from pathlib import Path
 
+from sqlalchemy import delete, select
 from sqlalchemy.exc import DBAPIError, OperationalError
 
 from ..celery_app import celery
+from ..models import ImageRecord
 from ..services import cleanup_service
 from ._sync_engine import sync_session_factory as _sync_session_factory
 
@@ -41,6 +43,206 @@ def delete_artist_cascade_task(self, *, artist_id: int) -> dict:
         )
 
 
+# Orphan repair (#859). Safety guard: an NFS/filesystem stall makes EVERY file
+# look missing — never delete records en masse on that basis. If a non-trivial
+# sample comes back mostly missing, ABORT without deleting (assume the FS is
+# unhealthy, not that the library evaporated). Operator-triggered ONLY — NOT a
+# periodic sweep, precisely to avoid an unattended run firing during an NFS blip.
+_ORPHAN_MIN_SAMPLE = 50
+_ORPHAN_MAX_MISSING_FRAC = 0.10
+
+
+@celery.task(
+    name="backend.app.tasks.admin.prune_missing_file_records_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def prune_missing_file_records_task(self) -> dict:
+    """Delete ImageRecords whose backing file is gone from disk (orphans — e.g.
+    the external-attach unlink bug #859). Every FK to image_record is CASCADE /
+    SET NULL, so a Core DELETE cleans provenance, series pages, predictions and
+    tag links; leftover thumbnails are unlinked best-effort. Returns a summary.
+
+    Aborts WITHOUT deleting if a non-trivial sample is mostly missing (a
+    filesystem/NFS stall, not real orphans) — see the guard constants above.
+    """
+    SessionLocal = _sync_session_factory()
+    checked = 0
+    missing_ids: list[int] = []
+    thumbs: list[str] = []
+    last_id = 0
+    with SessionLocal() as session:
+        while True:
+            rows = session.execute(
+                select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path)
+                .where(ImageRecord.id > last_id)
+                .order_by(ImageRecord.id)
+                .limit(1000)
+            ).all()
+            if not rows:
+                break
+            for rid, path, thumb in rows:
+                last_id = rid
+                checked += 1
+                if not (IMAGES_ROOT / path).exists():
+                    missing_ids.append(rid)
+                    if thumb:
+                        thumbs.append(thumb)
+
+        if not missing_ids:
+            return {"checked": checked, "missing": 0, "deleted": 0}
+
+        frac = len(missing_ids) / checked if checked else 0.0
+        if checked >= _ORPHAN_MIN_SAMPLE and frac > _ORPHAN_MAX_MISSING_FRAC:
+            log.warning(
+                "orphan-repair ABORTED: %d/%d (%.0f%%) records missing on disk — "
+                "likely a filesystem/NFS problem, not real orphans. No deletions.",
+                len(missing_ids), checked, frac * 100,
+            )
+            return {
+                "checked": checked, "missing": len(missing_ids), "deleted": 0,
+                "aborted": "too many missing — filesystem problem suspected",
+            }
+
+        deleted = 0
+        for i in range(0, len(missing_ids), 500):  # keep well under psycopg's param ceiling
+            chunk = missing_ids[i:i + 500]
+            session.execute(delete(ImageRecord).where(ImageRecord.id.in_(chunk)))
+            deleted += len(chunk)
+        session.commit()
+
+    for t in thumbs:  # cosmetic — outside the txn, never fail the repair on these
+        try:
+            (IMAGES_ROOT / t).unlink(missing_ok=True)
+        except OSError:
+            pass
+    log.info("orphan-repair: checked=%d missing=%d deleted=%d", checked, len(missing_ids), deleted)
+    return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
+
+
+@celery.task(
+    name="backend.app.tasks.admin.dedup_videos_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def dedup_videos_task(self, dry_run: bool = False) -> dict:
+    """Tier-1 video dedup (#871): re-probe NULL-duration videos, cluster by
+    artist + duration + aspect, keep the highest-res copy per cluster. dry_run
+    returns the projection (groups/redundant/reclaimable bytes) WITHOUT deleting;
+    apply re-points each loser's post links to the keeper then deletes the
+    redundant records + files. Operator-triggered; the summary lands in
+    task_run.metadata (FC-3i) for the Maintenance card to surface."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        return cleanup_service.dedup_videos(
+            session, images_root=IMAGES_ROOT, dry_run=dry_run,
+        )
+
+
+# Wall-clock budget for the gated-preview re-walk: stop walking new sources past
+# this and report partial (operator re-runs to finish). Sits under the soft limit
+# so the task returns its summary cleanly instead of being SIGKILLed mid-walk.
+_GATED_PURGE_BUDGET_SECONDS = 1500
+
+
+@celery.task(
+    name="backend.app.tasks.admin.purge_gated_previews_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def purge_gated_previews_task(self, dry_run: bool = True) -> dict:
+    """Cleanup (#874 follow-up): purge blurred locked-preview images grabbed from
+    tier-gated Patreon posts before the ingester fix. Re-walks every enabled
+    Patreon source (read-only, no downloads) to re-derive which posts are gated NOW
+    and the blurred filehashes Patreon serves for them, then matches by content
+    hash and (unless dry_run) deletes only those exact files — real content
+    downloaded when access existed has a different hash and is provably spared.
+
+    dry_run=True returns the projection (gated posts / matched images / reclaimable
+    bytes / unverifiable kept) WITHOUT deleting; dry_run=false applies it. The
+    summary lands in task_run.metadata (FC-3i) for the Maintenance card. Time-boxed
+    across sources: a `partial` result means re-run to finish (idempotent — a
+    re-walk re-derives the same matches and already-deleted files stay deleted)."""
+    import asyncio
+    import time as _time
+
+    from sqlalchemy import select as _select
+
+    from ..models import Source
+    from ..services.credential_crypto import CredentialCrypto
+    from ..services.credential_service import CredentialService
+    from ..services.patreon_client import PatreonClient
+    from ..services.patreon_resolver import resolve_campaign_id_for_source
+    from ._async_session import async_session_factory
+
+    key_path = IMAGES_ROOT / "secrets" / "credential_key.b64"
+
+    async def _run() -> dict:
+        async_factory, async_engine = async_session_factory()
+        SessionLocal = _sync_session_factory()
+        try:
+            async with async_factory() as async_session:
+                cred = CredentialService(async_session, CredentialCrypto(key_path))
+                cookies_obj = await cred.get_cookies_path("patreon")
+            cookies_path = str(cookies_obj) if cookies_obj else None
+
+            with SessionLocal() as session:
+                sources = session.execute(
+                    _select(Source.id, Source.url, Source.config_overrides)
+                    .where(Source.platform == "patreon", Source.enabled.is_(True))
+                ).all()
+
+            gated_map: dict[int, dict] = {}
+            scanned = 0
+            partial = False
+            loop = asyncio.get_running_loop()
+            start = _time.monotonic()
+            for sid, url, overrides in sources:
+                if _time.monotonic() - start >= _GATED_PURGE_BUDGET_SECONDS:
+                    partial = True
+                    break
+                campaign_id, _resolved = await resolve_campaign_id_for_source(
+                    url, cookies_path, overrides or {}
+                )
+                if not campaign_id:
+                    log.warning(
+                        "gated-purge: couldn't resolve campaign for source %s (%s) "
+                        "— skipping", sid, url,
+                    )
+                    continue
+                client = PatreonClient(cookies_path)
+                try:
+                    gated = await loop.run_in_executor(
+                        None, cleanup_service.collect_gated_previews, client, campaign_id
+                    )
+                except Exception as exc:  # one bad feed must not strand the sweep
+                    log.warning("gated-purge walk failed for source %s: %s", sid, exc)
+                    continue
+                scanned += 1
+                if gated:
+                    gated_map[int(sid)] = gated
+
+            with SessionLocal() as session:
+                summary = cleanup_service.purge_gated_previews(
+                    session, gated_map=gated_map, images_root=IMAGES_ROOT,
+                    dry_run=dry_run,
+                )
+            summary["sources_scanned"] = scanned
+            summary["sources_total"] = len(sources)
+            summary["partial"] = partial
+            return summary
+        finally:
+            await async_engine.dispose()
+
+    return asyncio.run(_run())
+
+
 @celery.task(
     name="backend.app.tasks.admin.bulk_delete_images_task",
     bind=True,
@@ -55,3 +257,155 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
         return cleanup_service.delete_images(
             session, image_ids=image_ids, images_root=IMAGES_ROOT,
         )
+
+
+# Time-box one chunk well under the soft limit so a large archive back-catalog
+# can't run the task into the Celery time limit (or hog the maintenance_long
+# lane). The task re-enqueues itself with the resume cursor until the scan is
+# exhausted — mirrors normalize_tags_task (operator-asked 2026-06-07: reasonable
+# timeout, then re-queue so other work keeps flowing).
+_REEXTRACT_CHUNK_SECONDS = 600
+
+
+@celery.task(
+    name="backend.app.tasks.admin.reextract_archive_attachments_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def reextract_archive_attachments_task(self, after_id: int = 0) -> dict:
+    """Wraps cleanup_service.reextract_archive_attachments (#713 part 2):
+    re-extract PostAttachments that are actually archives but were filed
+    opaquely before the magic-byte gate, and link their members to the post.
+
+    Time-boxed + self-resuming: scans attachments after ``after_id`` and, on a
+    chunk cut, re-enqueues from where it stopped so a big backlog finishes across
+    chunks instead of dying at the soft limit."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        summary = cleanup_service.reextract_archive_attachments(
+            session, images_root=IMAGES_ROOT,
+            time_budget_seconds=_REEXTRACT_CHUNK_SECONDS, after_id=after_id,
+        )
+    # More attachments past this chunk's cursor — continue in the next.
+    if summary.get("partial") and summary.get("resume_after_id", 0) > after_id:
+        log.info(
+            "reextract chunk done (%d scanned, %d archives, resume after id %s) "
+            "— re-enqueuing to continue",
+            summary.get("scanned", 0), summary.get("archives", 0),
+            summary["resume_after_id"],
+        )
+        reextract_archive_attachments_task.delay(summary["resume_after_id"])
+    return summary
+
+
+# Time-box one chunk well under the soft limit so a large back-catalog (the
+# first run recases the whole booru vocabulary) can't run the task into the
+# Celery time limit — it timed out at 40 min, operator-flagged 2026-06-07. The
+# task re-enqueues itself until nothing remains (idempotent — already-canonical
+# groups are skipped). 600s keeps each chunk short enough that the recovery
+# sweep and other maintenance tasks interleave on the concurrency-1 queue.
+_NORMALIZE_CHUNK_SECONDS = 600
+
+
+@celery.task(
+    name="backend.app.tasks.admin.normalize_tags_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def normalize_tags_task(self) -> dict:
+    """Wraps tag_service.normalize_existing_tags (#714): Title-Case the
+    back-catalog and merge case/whitespace-variant duplicate tags via the
+    tested async merge path. Time-boxed + self-resuming so a huge first run
+    finishes across chunks instead of timing out. Runs under its own asyncio
+    loop + per-task async engine (NullPool), mirroring download_source."""
+    import asyncio
+
+    from ..services.tag_service import normalize_existing_tags
+    from ._async_session import async_session_factory
+
+    async def _run() -> dict:
+        # lock_timeout=30s: a per-group merge repoints FKs across image_tag and
+        # series_page; if a statement blocks on a lock (e.g. behind a schema
+        # migration holding ACCESS EXCLUSIVE on series_page — the exact wedge that
+        # made this task run to the 40-min hard limit with no progress,
+        # operator-flagged 2026-06-07), it now fails fast. The per-group handler
+        # catches it (rollback + error++) and the loop continues, so one blocked
+        # group can't strand the whole chunk.
+        async_factory, async_engine = async_session_factory(
+            server_settings={"lock_timeout": "30s"}
+        )
+        try:
+            async with async_factory() as session:
+                # normalize_existing_tags commits per group internally.
+                return await normalize_existing_tags(
+                    session, dry_run=False,
+                    time_budget_seconds=_NORMALIZE_CHUNK_SECONDS,
+                )
+        finally:
+            await async_engine.dispose()
+
+    summary = asyncio.run(_run())
+    # More groups to canonicalize than fit this chunk — continue in the next.
+    if summary.get("partial") and summary.get("remaining", 0) > 0:
+        log.info(
+            "normalize_tags_task chunk done (%d processed, %d remaining) — "
+            "re-enqueuing to continue",
+            summary.get("groups_processed", 0), summary["remaining"],
+        )
+        normalize_tags_task.delay()
+    return summary
+
+
+# Time-box one rescan chunk well under the soft limit and re-enqueue from the
+# cursor — scoring every post against its artist's series is O(posts) and grows
+# with the library (FC-6.3). Mirrors normalize_tags_task.
+_SERIES_RESCAN_CHUNK_SECONDS = 600
+
+
+@celery.task(
+    name="backend.app.tasks.admin.rescan_series_suggestions_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict:
+    """Score posts against their artist's series and write pending suggestions
+    (FC-6.3). Settings-gated; time-boxed + self-resuming from a post-id cursor.
+    Per-task async engine (NullPool) under its own asyncio loop, like normalize."""
+    import asyncio
+
+    from ..models import ImportSettings
+    from ..services.series_match_service import SeriesMatchService
+    from ._async_session import async_session_factory
+
+    async def _run() -> dict:
+        async_factory, async_engine = async_session_factory()
+        try:
+            async with async_factory() as session:
+                settings = await ImportSettings.load(session)
+                if not settings.series_suggest_enabled:
+                    return {"skipped": "series suggestions disabled"}
+                threshold = settings.series_suggest_threshold
+                return await SeriesMatchService(session).rescan(
+                    threshold=threshold,
+                    time_budget_seconds=_SERIES_RESCAN_CHUNK_SECONDS,
+                    after_post_id=after_post_id,
+                )
+        finally:
+            await async_engine.dispose()
+
+    summary = asyncio.run(_run())
+    if summary.get("partial") and summary.get("resume_after_id", 0) > after_post_id:
+        log.info(
+            "rescan_series_suggestions chunk done (%d scanned, %d suggested, "
+            "resume after %s) — re-enqueuing",
+            summary.get("scanned", 0), summary.get("suggested", 0),
+            summary["resume_after_id"],
+        )
+        rescan_series_suggestions_task.delay(summary["resume_after_id"])
+    return summary

backend/app/tasks/backup.py

@@ -41,7 +41,11 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
     bind=True,
     autoretry_for=(OperationalError, DBAPIError),
     retry_backoff=10, retry_backoff_max=120, max_retries=2,
-    soft_time_limit=600, time_limit=720,
+    # A pg_dump can't be chunked; the 12-min limit timed out once the DB grew
+    # (operator-flagged 2026-06-07). 30/35 min gives real headroom. (A long
+    # backup still briefly holds the concurrency-1 maintenance lane — the
+    # structural fix is a dedicated lane for the long one-shots.)
+    soft_time_limit=1800, time_limit=2100,
 )
 def backup_db_task(self, *, tag: str | None = None,
                    triggered_by: str = "manual") -> dict:

backend/app/tasks/download.py

@@ -31,7 +31,7 @@ _KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
 # SoftTimeLimitExceeded in-process, whereas the HARD limit SIGKILLs the
 # worker (no chance to finalize). Both gallery-dl subprocess budgets
 # (gallery_dl.py: _DEFAULT_GDL_TIMEOUT_SECONDS=870 tick,
-# BACKFILL_TIMEOUT_SECONDS=1170 backfill) MUST sit below the soft limit
+# BACKFILL_CHUNK_SECONDS=600 per backfill chunk, plan #693) MUST sit below the soft limit
 # so subprocess.run raises its own TimeoutExpired first — that path
 # captures partial stdout/stderr and finalizes the DownloadEvent. soft is
 # max-subprocess (1170) + ~180s phase-3 persist headroom; hard is soft +
@@ -44,6 +44,25 @@ _KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
 DOWNLOAD_SOFT_TIME_LIMIT = 1350
 DOWNLOAD_HARD_TIME_LIMIT = 1500
 
+# Per-platform serialization (plan: concurrency cap). When a serialized
+# platform (Patreon) is already walking, defer this run by re-enqueuing it a
+# little later rather than holding a worker slot or bowling into the same rate
+# limit. Bounded so a wedged platform eventually runs anyway. The lock TTL sits
+# just past the hard kill so a SIGKILL'd worker's lock auto-expires; a backfill
+# chunk only holds it ~10 min, so the bounded wait stays well under the 30-min
+# DownloadEvent recovery sweep.
+_PLATFORM_LOCK_TTL = DOWNLOAD_HARD_TIME_LIMIT + 120
+_SERIALIZE_COUNTDOWN = 20
+_MAX_SERIALIZE_WAITS = 45  # ~15 min ceiling, then run uncapped as a safety valve
+
+
+def _peek_platform(source_id: int) -> str | None:
+    SyncFactory = _sync_session_factory()
+    with SyncFactory() as session:
+        return session.execute(
+            select(Source.platform).where(Source.id == source_id)
+        ).scalar_one_or_none()
+
 
 def _finalize_soft_limited(session: SyncSession, source_id: int) -> None:
     """Defense in depth for the soft-time-limit kill path.
@@ -75,10 +94,11 @@ def _finalize_soft_limited(session: SyncSession, source_id: int) -> None:
     ev.finished_at = now
     ev.error = (
         f"killed by Celery soft time limit ({DOWNLOAD_SOFT_TIME_LIMIT}s) "
-        "before the gallery-dl subprocess returned — the run exceeded its "
-        "budget and its stdout/stderr were lost with the worker thread. "
-        "If this recurs, the source is too large for one run; the backfill "
-        "budget was decremented so the next tick walks less."
+        "before the download finished — the run exceeded its time budget. "
+        "Progress is checkpointed per page, so the next tick resumes near "
+        "the cut; the backfill budget was decremented so it walks less. If "
+        "this recurs, a single post is heavy enough to overrun the chunk "
+        "time-box on its own."
     )
     ev.metadata_ = {
         **(ev.metadata_ or {}),
@@ -107,9 +127,48 @@ def _finalize_soft_limited(session: SyncSession, source_id: int) -> None:
     soft_time_limit=DOWNLOAD_SOFT_TIME_LIMIT,
     time_limit=DOWNLOAD_HARD_TIME_LIMIT,
 )
-def download_source(self, source_id: int) -> int:
+def download_source(self, source_id: int, _serialize_waits: int = 0) -> int:
     """Returns the DownloadEvent.id."""
 
+    # Per-platform concurrency cap: only one Patreon walk runs at a time.
+    from ..services.platform_lock import platform_lock
+
+    platform = _peek_platform(source_id)
+    lock = (
+        platform_lock(platform, ttl_seconds=_PLATFORM_LOCK_TTL)
+        if platform is not None
+        else None
+    )
+    if lock is not None:
+        try:
+            got_lock = bool(lock.acquire(blocking=False))
+        except Exception:  # noqa: BLE001 — broker hiccup → degrade to uncapped
+            log.warning("platform lock acquire failed for %s; running uncapped", platform)
+            lock = None
+            got_lock = True
+        if lock is not None and not got_lock:
+            if _serialize_waits < _MAX_SERIALIZE_WAITS:
+                # Another walk on this platform holds the lock — re-enqueue
+                # ourselves shortly (the pending DownloadEvent stays; no new
+                # event, no log spam) rather than running concurrently into
+                # the rate limit.
+                download_source.apply_async(
+                    (source_id,),
+                    {"_serialize_waits": _serialize_waits + 1},
+                    countdown=_SERIALIZE_COUNTDOWN,
+                )
+                log.info(
+                    "download_source(%s) deferred — %s already walking (wait %d/%d)",
+                    source_id, platform, _serialize_waits + 1, _MAX_SERIALIZE_WAITS,
+                )
+                return -1
+            log.warning(
+                "download_source(%s) running WITHOUT the %s lock — max serialize "
+                "waits hit, proceeding uncapped",
+                source_id, platform,
+            )
+            lock = None
+
     async def _run():
         async_factory, async_engine = async_session_factory()
         SyncFactory = _sync_session_factory()
@@ -143,6 +202,11 @@ def download_source(self, source_id: int) -> int:
                         gdl=gdl,
                         importer=importer,
                         cred_service=cred_service,
+                        # The native Patreon ingester opens its own short-lived
+                        # sync sessions for the seen-ledger (never held across
+                        # the walk). Same factory the importer's sync session
+                        # comes from — a different DB connection per checkout.
+                        sync_session_factory=SyncFactory,
                     )
                     return await svc.download_source(source_id)
         finally:
@@ -164,3 +228,12 @@ def download_source(self, source_id: int) -> int:
         except Exception:  # noqa: BLE001 — cleanup must not swallow the kill
             log.exception("soft-limit finalize failed for source %s", source_id)
         raise
+    finally:
+        # Release the per-platform lock so the next walk can proceed. The TTL is
+        # a backstop for a SIGKILL; here we free it the instant the run ends. A
+        # late release after TTL expiry raises (lock already gone) — ignore it.
+        if lock is not None:
+            try:
+                lock.release()
+            except Exception:  # noqa: BLE001 — TTL may have already freed it
+                pass

backend/app/tasks/external.py

@@ -0,0 +1,383 @@
+"""Celery worker for off-platform file-host downloads (external_link ledger).
+
+Walks the external_link table and fetches each pending/retryable link via
+external_fetch, then routes the downloaded file(s) through the existing importer
+(attach_in_place) so an archive becomes ImageRecords and any other file a
+PostAttachment — linked to the SAME post the link came from (via a synthesized
+sidecar). Long-running-task hygiene (rule #89): per-fetch wall-clock timeout,
+attempt tracking + dead-letter, a recovery sweep, and retention of dead rows.
+
+Concurrency: a per-host Redis lock serializes fetches to one-per-host (mega /
+gdrive ban-avoidance), and an atomic status claim (pending/failed →
+downloading) stops two workers grabbing the same link.
+
+Files land in the artist's library tree (so an attached art image stays in
+place, mirroring the gallery-dl download path); a captured archive/attachment is
+copied into its store and the on-disk original removed.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import redis
+from sqlalchemy import delete, select, update
+
+from ..celery_app import celery
+from ..config import get_config
+from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, Source
+from ..services.external_fetch import fetch_external
+from ..services.importer import Importer
+from ..services.thumbnailer import Thumbnailer
+from ._sync_engine import sync_session_factory as _sync_session_factory
+
+log = logging.getLogger(__name__)
+
+IMAGES_ROOT = Path("/images")
+
+# After this many failed attempts a link is dead-lettered (skipped by routine
+# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
+DEAD_LETTER_THRESHOLD = 3
+# Per-fetch read + total budgets now live in external_fetch (a short read
+# timeout fails a stalled host fast; a generous total caps a big-but-flowing
+# download). The celery soft/hard time_limit is the outer backstop above those.
+# Links enqueued per sweep — bounds the burst when a big backfill records many.
+_SWEEP_BATCH = 50
+# Dead rows older than this are pruned (retention).
+_RETENTION_DAYS = 30
+# Per-host serialize lock. TTL is a safety net for a worker that dies holding it
+# (normal completion/error releases it in `finally`); sized just past the fetch
+# total budget (30 min) so a dead worker can't wedge a host's links much longer
+# than one fetch would have taken.
+_LOCK_PREFIX = "fc:extdl_lock:"
+_LOCK_TTL = 2400
+_SERIALIZE_COUNTDOWN = 120
+_MAX_SERIALIZE_WAITS = 30
+
+_redis_client: redis.Redis | None = None
+
+
+def _redis() -> redis.Redis:
+    global _redis_client
+    if _redis_client is None:
+        _redis_client = redis.from_url(get_config().celery_broker_url)
+    return _redis_client
+
+
+def _host_enabled(settings: ImportSettings, host: str) -> bool:
+    """Per-host enable flag — defaults True (rule #26: works out of the box).
+    The Settings UI slice adds real columns; getattr keeps this forward-
+    compatible without a migration in this slice."""
+    return bool(getattr(settings, f"extdl_{host}_enabled", True))
+
+
+def _write_link_sidecar(file: Path, post: Post, platform: str, artist: Artist) -> None:
+    """Sidecar next to a fetched file so the importer links it to `post`
+    (category+id resolve the source+post; matches gallery-dl's sidecar shape)."""
+    data = {
+        "category": platform,
+        "id": post.external_post_id,
+        "url": post.post_url,
+        "title": post.post_title,
+        "artist": artist.name,
+    }
+    Path(str(file) + ".json").write_text(json.dumps(data))
+
+
+@celery.task(
+    name="backend.app.tasks.external.fetch_external_link",
+    bind=True,
+    soft_time_limit=3300,
+    time_limit=3600,
+)
+def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
+    """Fetch one external_link and route its file(s) through the importer."""
+    SessionLocal = _sync_session_factory()
+
+    # Atomic claim: only an actionable (pending/failed) row transitions to
+    # downloading, so a duplicate enqueue (sweep + post-download hook) no-ops.
+    with SessionLocal() as session:
+        claimed = session.execute(
+            update(ExternalLink)
+            .where(
+                ExternalLink.id == link_id,
+                ExternalLink.status.in_(("pending", "failed")),
+            )
+            .values(status="downloading")
+            .returning(ExternalLink.id)
+        ).first()
+        session.commit()
+        if claimed is None:
+            return {"link_id": link_id, "skipped": "not claimable"}
+
+        link = session.get(ExternalLink, link_id)
+        post = session.get(Post, link.post_id) if link.post_id else None
+        if post is None:
+            link.status = "dead"
+            link.last_error = "post missing"
+            link.completed_at = datetime.now(UTC)
+            session.commit()
+            return {"link_id": link_id, "error": "post missing"}
+        artist = session.get(Artist, post.artist_id)
+        source = session.get(Source, post.source_id) if post.source_id else None
+        platform = source.platform if source is not None else "patreon"
+        settings = ImportSettings.load_sync(session)
+
+        if not _host_enabled(settings, link.host):
+            log.info("extdl skip (host disabled): link=%s host=%s", link_id, link.host)
+            link.status = "skipped"
+            link.last_error = f"{link.host} disabled"
+            link.completed_at = datetime.now(UTC)
+            session.commit()
+            return {"link_id": link_id, "skipped": "host disabled"}
+
+        host, url, attempts = link.host, link.url, link.attempts
+        log.info(
+            "extdl start: link=%s host=%s post=%s artist=%s attempt=%d url=%s",
+            link_id, host, post.id, artist.slug, attempts + 1, url,
+        )
+
+    # Per-host serialize: one fetch per host at a time. If busy, requeue.
+    lock = None
+    try:
+        lock = _redis().lock(f"{_LOCK_PREFIX}{host}", timeout=_LOCK_TTL, blocking=False)
+        got = bool(lock.acquire(blocking=False))
+    except redis.RedisError as exc:
+        log.warning("extdl lock unavailable for %s: %s — running uncapped", host, exc)
+        lock, got = None, True
+    if not got:
+        # Release the claim back to pending so a later run can pick it up, then
+        # requeue with a countdown (bounded, like download_source).
+        with SessionLocal() as session:
+            session.execute(
+                update(ExternalLink).where(ExternalLink.id == link_id)
+                .values(status="pending")
+            )
+            session.commit()
+        log.info(
+            "extdl requeue (host %s busy): link=%s wait=%d", host, link_id, _serialize_waits,
+        )
+        if _serialize_waits < _MAX_SERIALIZE_WAITS:
+            fetch_external_link.apply_async(
+                (link_id,), {"_serialize_waits": _serialize_waits + 1},
+                countdown=_SERIALIZE_COUNTDOWN,
+            )
+        else:
+            log.warning("extdl giving up requeue (host %s busy): link=%s", host, link_id)
+        return {"link_id": link_id, "requeued": "host busy"}
+
+    started = time.monotonic()
+    # Per-LINK staging dir (not just per-post): two links on the same post (e.g.
+    # the same film from mega + gdrive) can emit the same filename — sharing one
+    # external/<post_id>/ dir let the second overwrite the first's file, then the
+    # dedup-skip unlink orphaned a record → 404 on playback. Isolating per link
+    # keeps each link's file at its own path. (#859)
+    post_dir = (
+        IMAGES_ROOT / artist.slug / platform / "external"
+        / str(post.external_post_id) / str(link_id)
+    )
+    try:
+        result = fetch_external(host, url, post_dir)
+        with SessionLocal() as session:
+            link = session.get(ExternalLink, link_id)
+            if not result.ok:
+                log.warning(
+                    "extdl fetch failed: link=%s host=%s error=%s",
+                    link_id, host, result.error,
+                )
+                _record_failure(link, attempts, result.error or "fetch failed")
+                session.commit()
+                return {"link_id": link_id, "error": result.error}
+
+            log.info(
+                "extdl fetched: link=%s host=%s files=%d bytes=%d",
+                link_id, host, len(result.files), result.bytes,
+            )
+            # Re-load post/artist/source attached to THIS session before handing
+            # them to the importer (the claim block's instances are detached).
+            post = session.get(Post, link.post_id)
+            artist = session.get(Artist, post.artist_id)
+            source = session.get(Source, post.source_id) if post.source_id else None
+            image_ids = _route_files(session, result.files, post, platform, artist, source)
+            link.status = "downloaded"
+            link.last_error = None
+            link.completed_at = datetime.now(UTC)
+            link.duration_seconds = time.monotonic() - started
+            session.commit()
+            log.info(
+                "extdl done: link=%s host=%s post=%s files=%d image(s)=%d dur=%.1fs",
+                link_id, host, post.id, len(result.files), len(image_ids),
+                link.duration_seconds,
+            )
+
+        # Thumbnails + ML for any newly-attached images (mirrors the download
+        # path). Lazy import to dodge a task-module import cycle.
+        if image_ids:
+            from .ml import tag_and_embed
+            from .thumbnail import generate_thumbnail
+            for img_id in image_ids:
+                generate_thumbnail.delay(img_id)
+                tag_and_embed.delay(img_id)
+        return {"link_id": link_id, "files": len(result.files), "images": len(image_ids)}
+    except Exception as exc:  # never leave a link stuck in 'downloading'
+        log.exception("external fetch task failed for link %s", link_id)
+        with SessionLocal() as session:
+            link = session.get(ExternalLink, link_id)
+            if link is not None and link.status == "downloading":
+                _record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
+                session.commit()
+        raise
+    finally:
+        if lock is not None:
+            try:
+                lock.release()
+            except Exception:  # noqa: BLE001 — TTL may have already freed it
+                pass
+
+
+def _record_failure(link: ExternalLink, attempts: int, error: str) -> None:
+    nxt = attempts + 1
+    link.attempts = nxt
+    link.last_error = error[:1000]
+    link.status = "dead" if nxt >= DEAD_LETTER_THRESHOLD else "failed"
+    link.completed_at = datetime.now(UTC)
+    log.warning(
+        "extdl link %s -> %s (attempt %d/%d): %s",
+        link.id, link.status, nxt, DEAD_LETTER_THRESHOLD, error[:200],
+    )
+
+
+def _route_files(session, files, post, platform, artist, source) -> list[int]:
+    """Import each fetched file through the SAME pipeline extracted-zip members
+    use — importer.attach_in_place — so a downloaded archive is extracted to
+    ImageRecords and provenance-linked to the post (via the synthesized sidecar),
+    a non-art file is captured as a PostAttachment, and an art image is attached
+    in place. Returns new image ids so the caller enqueues thumbnail + ML
+    (tagging) for them, exactly as download_service does for downloaded media.
+
+    Result handling mirrors download_service._phase3_persist branch-for-branch
+    (duplicate cleanup, the unextracted-archive warning of #718) and logs every
+    decision — this path is new and the operator expects to iterate on it."""
+    settings = ImportSettings.load_sync(session)
+    importer = Importer(
+        session=session,
+        images_root=IMAGES_ROOT,
+        import_root=IMAGES_ROOT,
+        thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
+        settings=settings,
+    )
+    image_ids: list[int] = []
+    for f in files:
+        _write_link_sidecar(f, post, platform, artist)
+        result = importer.attach_in_place(f, artist=artist, source=source)
+        session.commit()
+        if result.status in ("imported", "superseded"):
+            ids = list(getattr(result, "member_image_ids", []) or [])
+            if result.image_id is not None and result.image_id not in ids:
+                ids.append(result.image_id)
+            image_ids.extend(ids)
+            log.info(
+                "extdl import: post=%s file=%s status=%s -> %d image(s) "
+                "(provenance-linked, queued for tagging)",
+                post.id, f.name, result.status, len(ids),
+            )
+        elif result.status == "attached":
+            # Non-art / archive captured as a PostAttachment (copied into the
+            # store) — drop the on-disk original. An archive captured WITHOUT
+            # extracting any image carries the reason (the recurring "zip but no
+            # images" symptom, #718) — surface it loudly.
+            if result.error:
+                log.warning(
+                    "extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s",
+                    post.id, f.name, result.error,
+                )
+            else:
+                log.info("extdl attachment captured: post=%s file=%s", post.id, f.name)
+            f.unlink(missing_ok=True)
+        elif result.status == "skipped":
+            reason = result.skip_reason.value if result.skip_reason else None
+            log.info(
+                "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
+            )
+            if reason in ("duplicate_hash", "duplicate_phash"):
+                # Path-safe unlink: external files are imported IN PLACE, so `f`
+                # can BE the canonical record's backing file (same content
+                # re-fetched / a colliding name). Only delete `f` when it is NOT
+                # the existing record's file — otherwise we orphan the record and
+                # playback 404s. (#859)
+                canonical = None
+                if result.image_id is not None:
+                    rec = session.get(ImageRecord, result.image_id)
+                    if rec is not None:
+                        canonical = (IMAGES_ROOT / rec.path).resolve()
+                if canonical != f.resolve():
+                    f.unlink(missing_ok=True)  # a redundant copy — safe to drop
+                else:
+                    log.info(
+                        "extdl keep: %s IS record %s's canonical file — not unlinking",
+                        f.name, result.image_id,
+                    )
+        elif result.status == "failed":
+            log.warning(
+                "extdl import FAILED: post=%s file=%s error=%s",
+                post.id, f.name, result.error,
+            )
+            f.unlink(missing_ok=True)
+        else:  # 'refreshed' or any future status — work happened, no new images
+            log.info(
+                "extdl import: post=%s file=%s status=%s", post.id, f.name, result.status,
+            )
+        # The synthesized sidecar has done its job — don't litter the tree.
+        Path(str(f) + ".json").unlink(missing_ok=True)
+    return image_ids
+
+
+@celery.task(name="backend.app.tasks.external.sweep_external_links")
+def sweep_external_links() -> dict:
+    """Enqueue a bounded batch of actionable links (pending, or failed below the
+    dead-letter threshold). Driven by the post-download hook and a beat tick."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        ids = session.execute(
+            select(ExternalLink.id).where(
+                ExternalLink.status.in_(("pending", "failed")),
+                ExternalLink.attempts < DEAD_LETTER_THRESHOLD,
+            ).order_by(ExternalLink.id.asc()).limit(_SWEEP_BATCH)
+        ).scalars().all()
+    for link_id in ids:
+        fetch_external_link.delay(link_id)
+    return {"enqueued": len(ids)}
+
+
+@celery.task(name="backend.app.tasks.external.recover_external_links")
+def recover_external_links() -> dict:
+    """Recovery sweep (rule #89): reset dead links back to retryable so a stuck
+    host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        session.execute(
+            update(ExternalLink).where(ExternalLink.status == "dead")
+            .values(status="failed", attempts=0, last_error=None)
+        )
+        session.commit()
+    return sweep_external_links()
+
+
+@celery.task(name="backend.app.tasks.external.prune_external_links")
+def prune_external_links() -> dict:
+    """Retention (rule #89): delete long-dead links so the ledger doesn't grow
+    unbounded. Downloaded links are kept (they're the record of what we have)."""
+    cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        res = session.execute(
+            delete(ExternalLink).where(
+                ExternalLink.status == "dead",
+                ExternalLink.created_at < cutoff,
+            )
+        )
+        session.commit()
+    return {"pruned": res.rowcount or 0}