feat(maintenance): reconcile duplicate posts (gallery-dl→native unify)

An artist first downloaded by gallery-dl gets Post rows keyed by the per-
attachment id; a later native walk keys the SAME real post by the post id. They
never dedup (uq_post_source_external_id is on external_post_id) → duplicate post
rows (cheunart: 943→1109). The real post id is recoverable in-DB from
raw_metadata['post_id'] (both eras store the sidecar there).

reconcile_duplicate_posts (cleanup_service): group posts by (source_id, canonical
post_id = raw_metadata.post_id else external_post_id); for each group >1, keep the
row already keyed by the post id (the format the CURRENT native downloader
produces, so future walks dedup and this can't recur), re-point
ImageRecord.primary_post_id / ImageProvenance / PostAttachment / ExternalLink onto
it conflict-safe (drop the loser's row where the keeper already has the equivalent,
per each table's uniqueness), backfill the keeper's empty date/title/body/raw_meta
from a loser, set external_post_id=post_id + derive post_url, delete losers.
IMAGES ARE NOT TOUCHED (content-addressed/deduped already; operator-confirmed).

Preview/apply share find_duplicate_post_groups (rule 93). API
/api/admin/posts/reconcile-duplicates (dry_run→{groups,posts_to_merge,sample};
apply→{groups,merged,sample}; optional source_id). UI: a second section on
PostMaintenanceCard (preview groups+sample → confirm merge). Tests: merge +
metadata backfill + image move, no-op when unique, provenance-collision dedup.

Design: milestone #73. Forensics: note #917. Out of scope (flagged): cheunart vs
Cheunart case-variant artist dirs/rows.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/api/admin.py

@@ -226,6 +226,33 @@ async def posts_prune_bare():
     return jsonify(result)
 
 
+@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"])
+async def posts_reconcile_duplicates():
+    """Tier-A: unify duplicate post rows for the same real post — the gallery-dl
+    (attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper,
+    moving image/provenance/attachment/link rows over. Images are untouched.
+    dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies
+    and returns {groups, merged, sample}. Optional source_id scopes to one source.
+    Same find_duplicate_post_groups predicate drives preview + apply (rule 93)."""
+    from ..services.cleanup_service import reconcile_duplicate_posts
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", False))
+    raw_source = body.get("source_id")
+    try:
+        source_id = int(raw_source) if raw_source is not None else None
+    except (TypeError, ValueError):
+        return _bad("invalid_source_id", detail="source_id must be an integer")
+
+    async with get_session() as session:
+        result = await session.run_sync(
+            lambda sync_sess: reconcile_duplicate_posts(
+                sync_sess, source_id=source_id, dry_run=dry_run,
+            )
+        )
+    return jsonify(result)
+
+
 @admin_bp.route("/tags/purge-legacy", methods=["POST"])
 async def tags_purge_legacy():
     """Tier-A: delete legacy IR-migration tags — archive/post/artist

backend/app/services/cleanup_service.py

@@ -22,6 +22,7 @@ from sqlalchemy.orm import Session
 
 from ..models import (
     Artist,
+    ExternalLink,
     ImageProvenance,
     ImageRecord,
     LibraryAuditRun,
@@ -36,6 +37,7 @@ from ..models.series_page import SeriesPage
 from ..models.tag import image_tag
 from ..utils import safe_probe
 from .importer import _VIDEO_DUP_ASPECT_TOL, _VIDEO_DUP_DURATION_TOL_SECONDS
+from .platforms import PLATFORMS
 
 log = logging.getLogger(__name__)
 
@@ -507,6 +509,182 @@ def prune_bare_posts(session: Session, *, dry_run: bool = False) -> dict:
     return {"deleted": result.rowcount or 0, "sample_names": sample}
 
 
+# -- duplicate-post reconciliation (gallery-dl → native migration) ----------
+# An artist first downloaded by gallery-dl gets Post rows keyed by the per-
+# ATTACHMENT id (gallery-dl's `id`); a later native walk keys the SAME real post
+# by the post id. The two never dedup (uq_post_source_external_id is on
+# external_post_id) → duplicate post rows. The real post id is recoverable in-DB
+# from raw_metadata["post_id"] (both eras store the sidecar there). We unify each
+# group onto ONE post row keyed the way the CURRENT native downloader keys it
+# (post id), so future native walks match and the dup can't recur. Images are
+# untouched (content-addressed/deduped already); only post rows + their link
+# rows move. Milestone #73 / note #917.
+
+
+def _canonical_post_id(post: Post) -> str | None:
+    """The real platform post id used to group duplicate rows: raw_metadata
+    ['post_id'] when present (the true id, stored by both gallery-dl and native
+    imports), else external_post_id. None when neither is usable."""
+    rm = post.raw_metadata or {}
+    pid = rm.get("post_id")
+    if pid is not None and str(pid).strip():
+        return str(pid)
+    return post.external_post_id or None
+
+
+def find_duplicate_post_groups(
+    session: Session, *, source_id: int | None = None,
+) -> list[list[Post]]:
+    """Groups of >1 Post that are the SAME real post (same source_id + canonical
+    post id) — the gallery-dl(attachment-id) / native(post-id) duplicates. Shared
+    by the dry-run preview and the live reconcile (preview/apply parity)."""
+    stmt = select(Post)
+    if source_id is not None:
+        stmt = stmt.where(Post.source_id == source_id)
+    groups: dict[tuple, list[Post]] = {}
+    for post in session.execute(stmt.order_by(Post.id)).scalars().all():
+        cpid = _canonical_post_id(post)
+        if not cpid:
+            continue
+        groups.setdefault((post.source_id, cpid), []).append(post)
+    return [posts for posts in groups.values() if len(posts) > 1]
+
+
+def _choose_keeper(posts: list[Post], cpid: str) -> Post:
+    """The surviving row for a dup group: prefer one already keyed by the
+    canonical post id (the native format we keep), then the most complete
+    (has description, has date), then the lowest id for stability."""
+    native = [p for p in posts if p.external_post_id == cpid]
+    pool = native or posts
+    return min(pool, key=lambda p: (not p.description, not p.post_date, p.id))
+
+
+def _repoint_post_links(session: Session, loser_id: int, keeper_id: int) -> None:
+    """Move every link row from a loser post to the keeper, conflict-safe against
+    each table's uniqueness (drop the loser's row when the keeper already has the
+    equivalent, else re-point). Images themselves are never touched."""
+    # ImageRecord.primary_post_id — no uniqueness; straight re-point.
+    session.execute(
+        update(ImageRecord)
+        .where(ImageRecord.primary_post_id == loser_id)
+        .values(primary_post_id=keeper_id)
+    )
+    # ImageProvenance — unique (image_record_id, post_id).
+    dup_imgs = select(ImageProvenance.image_record_id).where(
+        ImageProvenance.post_id == keeper_id
+    )
+    session.execute(
+        delete(ImageProvenance).where(
+            ImageProvenance.post_id == loser_id,
+            ImageProvenance.image_record_id.in_(dup_imgs),
+        )
+    )
+    session.execute(
+        update(ImageProvenance)
+        .where(ImageProvenance.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+    # PostAttachment — partial unique (post_id, sha256) where post_id NOT NULL.
+    dup_shas = select(PostAttachment.sha256).where(PostAttachment.post_id == keeper_id)
+    session.execute(
+        delete(PostAttachment).where(
+            PostAttachment.post_id == loser_id,
+            PostAttachment.sha256.in_(dup_shas),
+        )
+    )
+    session.execute(
+        update(PostAttachment)
+        .where(PostAttachment.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+    # ExternalLink — unique (post_id, url).
+    dup_urls = select(ExternalLink.url).where(ExternalLink.post_id == keeper_id)
+    session.execute(
+        delete(ExternalLink).where(
+            ExternalLink.post_id == loser_id,
+            ExternalLink.url.in_(dup_urls),
+        )
+    )
+    session.execute(
+        update(ExternalLink)
+        .where(ExternalLink.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+
+
+def _fill_missing_post_fields(keeper: Post, loser: Post) -> None:
+    """Backfill the keeper's empty metadata from a loser that has it (the native
+    stub is often bare; the gallery-dl row carries date/title/body/raw_metadata)."""
+    if not keeper.post_date and loser.post_date:
+        keeper.post_date = loser.post_date
+    if not keeper.post_title and loser.post_title:
+        keeper.post_title = loser.post_title
+    if not keeper.description and loser.description:
+        keeper.description = loser.description
+    if not keeper.raw_metadata and loser.raw_metadata:
+        keeper.raw_metadata = loser.raw_metadata
+    if not keeper.attachment_count and loser.attachment_count:
+        keeper.attachment_count = loser.attachment_count
+
+
+def _canonical_post_url(post: Post, cpid: str) -> str | None:
+    """Permalink for the unified post, via the platform's derive_post_url hook
+    (subscribestar/patreon synthesize `…/posts/<id>`). Falls back to the keeper's
+    existing url when no hook applies."""
+    platform = (post.raw_metadata or {}).get("category")
+    info = PLATFORMS.get(platform) if platform else None
+    if info is not None and info.derive_post_url is not None:
+        derived = info.derive_post_url({"post_id": cpid})
+        if derived:
+            return derived
+    return post.post_url
+
+
+def reconcile_duplicate_posts(
+    session: Session, *, source_id: int | None = None, dry_run: bool = False,
+) -> dict:
+    """Unify duplicate post rows (gallery-dl attachment-id + native post-id) onto
+    one keeper per real post, re-keyed to the post id. Images untouched.
+
+    Returns:
+      dry_run=True:  {"groups": G, "posts_to_merge": L, "sample": [...]}
+      dry_run=False: {"groups": G, "merged": L, "sample": [...]}
+    where L = rows that would be (were) deleted after merging into keepers. The
+    SAME find_duplicate_post_groups predicate drives preview and apply (rule 93).
+    """
+    groups = find_duplicate_post_groups(session, source_id=source_id)
+    sample: list[dict] = []
+    losers_total = 0
+    for posts in groups:
+        cpid = _canonical_post_id(posts[0])
+        keeper = _choose_keeper(posts, cpid)
+        losers = [p for p in posts if p.id != keeper.id]
+        losers_total += len(losers)
+        if len(sample) < 50:
+            sample.append({
+                "post_id": cpid,
+                "rows": len(posts),
+                "keeper_id": keeper.id,
+                "title": keeper.post_title or f"Post {cpid}",
+            })
+        if dry_run:
+            continue
+        for loser in losers:
+            _repoint_post_links(session, loser.id, keeper.id)
+            _fill_missing_post_fields(keeper, loser)
+        keeper.external_post_id = cpid
+        new_url = _canonical_post_url(keeper, cpid)
+        if new_url:
+            keeper.post_url = new_url
+        session.flush()
+        for loser in losers:
+            session.delete(loser)
+    if dry_run:
+        return {"groups": len(groups), "posts_to_merge": losers_total, "sample": sample}
+    session.commit()
+    return {"groups": len(groups), "merged": losers_total, "sample": sample}
+
+
 # Legacy tags FC no longer uses, in two shapes:
 #   (1) kinds the tag input never produces — archive/post/artist.
 #       provenance (post grouping) + archive membership are their own