Merge pull request 'feat(maintenance): reconcile duplicate posts (gallery-dl→native unify)' (#124) from dev into main

backend/app/api/admin.py

@@ -226,6 +226,33 @@ async def posts_prune_bare():
     return jsonify(result)
 
 
+@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"])
+async def posts_reconcile_duplicates():
+    """Tier-A: unify duplicate post rows for the same real post — the gallery-dl
+    (attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper,
+    moving image/provenance/attachment/link rows over. Images are untouched.
+    dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies
+    and returns {groups, merged, sample}. Optional source_id scopes to one source.
+    Same find_duplicate_post_groups predicate drives preview + apply (rule 93)."""
+    from ..services.cleanup_service import reconcile_duplicate_posts
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", False))
+    raw_source = body.get("source_id")
+    try:
+        source_id = int(raw_source) if raw_source is not None else None
+    except (TypeError, ValueError):
+        return _bad("invalid_source_id", detail="source_id must be an integer")
+
+    async with get_session() as session:
+        result = await session.run_sync(
+            lambda sync_sess: reconcile_duplicate_posts(
+                sync_sess, source_id=source_id, dry_run=dry_run,
+            )
+        )
+    return jsonify(result)
+
+
 @admin_bp.route("/tags/purge-legacy", methods=["POST"])
 async def tags_purge_legacy():
     """Tier-A: delete legacy IR-migration tags — archive/post/artist

backend/app/services/cleanup_service.py

@@ -22,6 +22,7 @@ from sqlalchemy.orm import Session
 
 from ..models import (
     Artist,
+    ExternalLink,
     ImageProvenance,
     ImageRecord,
     LibraryAuditRun,
@@ -36,6 +37,7 @@ from ..models.series_page import SeriesPage
 from ..models.tag import image_tag
 from ..utils import safe_probe
 from .importer import _VIDEO_DUP_ASPECT_TOL, _VIDEO_DUP_DURATION_TOL_SECONDS
+from .platforms import PLATFORMS
 
 log = logging.getLogger(__name__)
 
@@ -507,6 +509,182 @@ def prune_bare_posts(session: Session, *, dry_run: bool = False) -> dict:
     return {"deleted": result.rowcount or 0, "sample_names": sample}
 
 
+# -- duplicate-post reconciliation (gallery-dl → native migration) ----------
+# An artist first downloaded by gallery-dl gets Post rows keyed by the per-
+# ATTACHMENT id (gallery-dl's `id`); a later native walk keys the SAME real post
+# by the post id. The two never dedup (uq_post_source_external_id is on
+# external_post_id) → duplicate post rows. The real post id is recoverable in-DB
+# from raw_metadata["post_id"] (both eras store the sidecar there). We unify each
+# group onto ONE post row keyed the way the CURRENT native downloader keys it
+# (post id), so future native walks match and the dup can't recur. Images are
+# untouched (content-addressed/deduped already); only post rows + their link
+# rows move. Milestone #73 / note #917.
+
+
+def _canonical_post_id(post: Post) -> str | None:
+    """The real platform post id used to group duplicate rows: raw_metadata
+    ['post_id'] when present (the true id, stored by both gallery-dl and native
+    imports), else external_post_id. None when neither is usable."""
+    rm = post.raw_metadata or {}
+    pid = rm.get("post_id")
+    if pid is not None and str(pid).strip():
+        return str(pid)
+    return post.external_post_id or None
+
+
+def find_duplicate_post_groups(
+    session: Session, *, source_id: int | None = None,
+) -> list[list[Post]]:
+    """Groups of >1 Post that are the SAME real post (same source_id + canonical
+    post id) — the gallery-dl(attachment-id) / native(post-id) duplicates. Shared
+    by the dry-run preview and the live reconcile (preview/apply parity)."""
+    stmt = select(Post)
+    if source_id is not None:
+        stmt = stmt.where(Post.source_id == source_id)
+    groups: dict[tuple, list[Post]] = {}
+    for post in session.execute(stmt.order_by(Post.id)).scalars().all():
+        cpid = _canonical_post_id(post)
+        if not cpid:
+            continue
+        groups.setdefault((post.source_id, cpid), []).append(post)
+    return [posts for posts in groups.values() if len(posts) > 1]
+
+
+def _choose_keeper(posts: list[Post], cpid: str) -> Post:
+    """The surviving row for a dup group: prefer one already keyed by the
+    canonical post id (the native format we keep), then the most complete
+    (has description, has date), then the lowest id for stability."""
+    native = [p for p in posts if p.external_post_id == cpid]
+    pool = native or posts
+    return min(pool, key=lambda p: (not p.description, not p.post_date, p.id))
+
+
+def _repoint_post_links(session: Session, loser_id: int, keeper_id: int) -> None:
+    """Move every link row from a loser post to the keeper, conflict-safe against
+    each table's uniqueness (drop the loser's row when the keeper already has the
+    equivalent, else re-point). Images themselves are never touched."""
+    # ImageRecord.primary_post_id — no uniqueness; straight re-point.
+    session.execute(
+        update(ImageRecord)
+        .where(ImageRecord.primary_post_id == loser_id)
+        .values(primary_post_id=keeper_id)
+    )
+    # ImageProvenance — unique (image_record_id, post_id).
+    dup_imgs = select(ImageProvenance.image_record_id).where(
+        ImageProvenance.post_id == keeper_id
+    )
+    session.execute(
+        delete(ImageProvenance).where(
+            ImageProvenance.post_id == loser_id,
+            ImageProvenance.image_record_id.in_(dup_imgs),
+        )
+    )
+    session.execute(
+        update(ImageProvenance)
+        .where(ImageProvenance.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+    # PostAttachment — partial unique (post_id, sha256) where post_id NOT NULL.
+    dup_shas = select(PostAttachment.sha256).where(PostAttachment.post_id == keeper_id)
+    session.execute(
+        delete(PostAttachment).where(
+            PostAttachment.post_id == loser_id,
+            PostAttachment.sha256.in_(dup_shas),
+        )
+    )
+    session.execute(
+        update(PostAttachment)
+        .where(PostAttachment.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+    # ExternalLink — unique (post_id, url).
+    dup_urls = select(ExternalLink.url).where(ExternalLink.post_id == keeper_id)
+    session.execute(
+        delete(ExternalLink).where(
+            ExternalLink.post_id == loser_id,
+            ExternalLink.url.in_(dup_urls),
+        )
+    )
+    session.execute(
+        update(ExternalLink)
+        .where(ExternalLink.post_id == loser_id)
+        .values(post_id=keeper_id)
+    )
+
+
+def _fill_missing_post_fields(keeper: Post, loser: Post) -> None:
+    """Backfill the keeper's empty metadata from a loser that has it (the native
+    stub is often bare; the gallery-dl row carries date/title/body/raw_metadata)."""
+    if not keeper.post_date and loser.post_date:
+        keeper.post_date = loser.post_date
+    if not keeper.post_title and loser.post_title:
+        keeper.post_title = loser.post_title
+    if not keeper.description and loser.description:
+        keeper.description = loser.description
+    if not keeper.raw_metadata and loser.raw_metadata:
+        keeper.raw_metadata = loser.raw_metadata
+    if not keeper.attachment_count and loser.attachment_count:
+        keeper.attachment_count = loser.attachment_count
+
+
+def _canonical_post_url(post: Post, cpid: str) -> str | None:
+    """Permalink for the unified post, via the platform's derive_post_url hook
+    (subscribestar/patreon synthesize `…/posts/<id>`). Falls back to the keeper's
+    existing url when no hook applies."""
+    platform = (post.raw_metadata or {}).get("category")
+    info = PLATFORMS.get(platform) if platform else None
+    if info is not None and info.derive_post_url is not None:
+        derived = info.derive_post_url({"post_id": cpid})
+        if derived:
+            return derived
+    return post.post_url
+
+
+def reconcile_duplicate_posts(
+    session: Session, *, source_id: int | None = None, dry_run: bool = False,
+) -> dict:
+    """Unify duplicate post rows (gallery-dl attachment-id + native post-id) onto
+    one keeper per real post, re-keyed to the post id. Images untouched.
+
+    Returns:
+      dry_run=True:  {"groups": G, "posts_to_merge": L, "sample": [...]}
+      dry_run=False: {"groups": G, "merged": L, "sample": [...]}
+    where L = rows that would be (were) deleted after merging into keepers. The
+    SAME find_duplicate_post_groups predicate drives preview and apply (rule 93).
+    """
+    groups = find_duplicate_post_groups(session, source_id=source_id)
+    sample: list[dict] = []
+    losers_total = 0
+    for posts in groups:
+        cpid = _canonical_post_id(posts[0])
+        keeper = _choose_keeper(posts, cpid)
+        losers = [p for p in posts if p.id != keeper.id]
+        losers_total += len(losers)
+        if len(sample) < 50:
+            sample.append({
+                "post_id": cpid,
+                "rows": len(posts),
+                "keeper_id": keeper.id,
+                "title": keeper.post_title or f"Post {cpid}",
+            })
+        if dry_run:
+            continue
+        for loser in losers:
+            _repoint_post_links(session, loser.id, keeper.id)
+            _fill_missing_post_fields(keeper, loser)
+        keeper.external_post_id = cpid
+        new_url = _canonical_post_url(keeper, cpid)
+        if new_url:
+            keeper.post_url = new_url
+        session.flush()
+        for loser in losers:
+            session.delete(loser)
+    if dry_run:
+        return {"groups": len(groups), "posts_to_merge": losers_total, "sample": sample}
+    session.commit()
+    return {"groups": len(groups), "merged": losers_total, "sample": sample}
+
+
 # Legacy tags FC no longer uses, in two shapes:
 #   (1) kinds the tag input never produces — archive/post/artist.
 #       provenance (post grouping) + archive membership are their own

frontend/src/components/settings/PostMaintenanceCard.vue

@@ -49,12 +49,56 @@
           class="ml-3 text-caption text-success"
         >Deleted {{ deleted }} ✓</span>
       </div>
+
+      <v-divider class="my-5" />
+
+      <p class="fc-muted text-body-2 mb-4">
+        Unify <strong>duplicate posts</strong> — when an artist was first
+        downloaded by gallery-dl and later re-walked by the native ingester, the
+        same post can exist twice (once keyed by the file's attachment id, once by
+        the real post id). This merges each pair onto one canonical post (keeping
+        the native post-id key), moving images, attachments and links onto the
+        survivor. Images themselves are never touched.
+      </p>
+
+      <v-btn
+        color="accent" variant="flat" rounded="pill"
+        prepend-icon="mdi-magnify"
+        :loading="loadingDupPreview"
+        class="mb-3"
+        @click="onPreviewDupes"
+      >Preview duplicate posts</v-btn>
+
+      <div v-if="dupPreview">
+        <p class="text-body-2 mb-2">
+          <strong>{{ dupPreview.groups }}</strong> duplicate group(s),
+          <strong>{{ dupPreview.posts_to_merge }}</strong> redundant row(s) to
+          merge.
+          <span v-if="dupPreview.groups > 50" class="fc-muted">Showing first 50.</span>
+          <span v-else-if="!dupPreview.groups" class="fc-muted">No duplicates found.</span>
+        </p>
+        <SampleNameGrid
+          v-if="dupSampleNames.length"
+          :names="dupSampleNames" class="mb-3"
+        />
+        <v-btn
+          color="error" variant="flat" rounded="pill"
+          prepend-icon="mdi-merge"
+          :disabled="!dupPreview.groups"
+          :loading="merging"
+          @click="onMergeDupes"
+        >Merge {{ dupPreview.posts_to_merge }} duplicate(s)</v-btn>
+        <span
+          v-if="merged != null"
+          class="ml-3 text-caption text-success"
+        >Merged {{ merged }} ✓</span>
+      </div>
     </v-card-text>
   </v-card>
 </template>
 
 <script setup>
-import { ref } from 'vue'
+import { computed, ref } from 'vue'
 
 import { useAdminStore } from '../../stores/admin.js'
 import SampleNameGrid from '../common/SampleNameGrid.vue'
@@ -66,6 +110,16 @@ const loadingPreview = ref(false)
 const committing = ref(false)
 const deleted = ref(null)
 
+const dupPreview = ref(null)
+const loadingDupPreview = ref(false)
+const merging = ref(false)
+const merged = ref(null)
+const dupSampleNames = computed(() =>
+  (dupPreview.value?.sample ?? []).map(
+    (g) => `${g.title} — ${g.rows} rows`,
+  ),
+)
+
 async function onPreview() {
   loadingPreview.value = true
   deleted.value = null
@@ -88,6 +142,28 @@ async function onCommit() {
     committing.value = false
   }
 }
+
+async function onPreviewDupes() {
+  loadingDupPreview.value = true
+  merged.value = null
+  try {
+    dupPreview.value = await store.reconcileDuplicatePosts({ dryRun: true })
+  } finally {
+    loadingDupPreview.value = false
+  }
+}
+
+async function onMergeDupes() {
+  merging.value = true
+  try {
+    const result = await store.reconcileDuplicatePosts({ dryRun: false })
+    merged.value = result.merged ?? 0
+    // Same predicate as the preview — after the merge there are no dup groups left.
+    dupPreview.value = { groups: 0, posts_to_merge: 0, sample: [] }
+  } finally {
+    merging.value = false
+  }
+}
 </script>
 
 <style scoped>

frontend/src/stores/admin.js

@@ -129,6 +129,22 @@ export const useAdminStore = defineStore('admin', () => {
     }
   }
 
+  // Tier-A: unify duplicate post rows (gallery-dl attachment-id + native post-id
+  // for the same real post) onto one post-id-keyed keeper. Images untouched.
+  // Preview/apply parity on the backend; UI previews then confirms.
+  async function reconcileDuplicatePosts({ dryRun = true, sourceId = null } = {}) {
+    lastError.value = null
+    try {
+      return await api.post(
+        '/api/admin/posts/reconcile-duplicates',
+        { body: { dry_run: dryRun, source_id: sourceId } },
+      )
+    } catch (e) {
+      lastError.value = e.message
+      throw e
+    }
+  }
+
   async function purgeLegacyTags({ dryRun = true } = {}) {
     lastError.value = null
     try {
@@ -208,6 +224,7 @@ export const useAdminStore = defineStore('admin', () => {
     tagUsageCount,
     pruneUnusedTags,
     pruneBarePosts,
+    reconcileDuplicatePosts,
     purgeLegacyTags,
     resetContentTagging,
     normalizeTags,

tests/test_cleanup_service.py

@@ -5,6 +5,8 @@ side effects use tmp_path. Assertions on mutated rows use COLUMN
 SELECTS per reference_async_coredml_test_assertions — never
 re-read ORM attributes after a service mutates and re-fetches.
 """
+from datetime import UTC, datetime
+
 import pytest
 from sqlalchemy import func, select
 
@@ -767,3 +769,87 @@ def test_purge_gated_previews_no_gated_posts_is_noop(db_sync, tmp_path):
     assert out["matched"] == 0
     assert out["deleted"] == 0
     assert out["gated_posts"] == 0
+
+
+# ---- duplicate-post reconciliation (gallery-dl → native, milestone #73) ----
+
+
+def test_reconcile_merges_attachmentid_and_postid_dupes(db_sync, tmp_path):
+    # gallery-dl row (keyed by attachment id, holds the image + body/date) +
+    # native row (keyed by the real post id, bare) → unified onto ONE post-id-
+    # keyed keeper, image moved over, keeper backfilled from the legacy row.
+    a = _make_artist(db_sync, slug="rc1")
+    img = _make_image(db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="c" * 64)
+    legacy = Post(
+        artist_id=a.id, external_post_id="711509",
+        raw_metadata={"post_id": 1923726, "category": "subscribestar"},
+        description="real body", post_title="T",
+        post_date=datetime(2025, 6, 20, tzinfo=UTC),
+    )
+    native = Post(
+        artist_id=a.id, external_post_id="1923726",
+        raw_metadata={"post_id": 1923726, "category": "subscribestar"},
+    )
+    db_sync.add_all([legacy, native])
+    db_sync.flush()
+    img.primary_post_id = legacy.id
+    img_id, native_id, legacy_id = img.id, native.id, legacy.id
+    db_sync.commit()
+
+    prev = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=True)
+    assert prev["groups"] == 1
+    assert prev["posts_to_merge"] == 1
+    res = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
+    assert res["merged"] == 1
+
+    rows = db_sync.execute(select(Post.id, Post.external_post_id)).all()
+    assert len(rows) == 1
+    keeper_id, keeper_epid = rows[0]
+    assert keeper_id == native_id  # native (post-id-keyed) row survives
+    assert keeper_epid == "1923726"
+    # image moved to keeper; legacy row gone; keeper backfilled.
+    assert db_sync.execute(
+        select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
+    ).scalar_one() == native_id
+    assert db_sync.execute(
+        select(func.count()).select_from(Post).where(Post.id == legacy_id)
+    ).scalar_one() == 0
+    desc, pdate = db_sync.execute(
+        select(Post.description, Post.post_date).where(Post.id == native_id)
+    ).one()
+    assert desc == "real body"
+    assert pdate is not None
+
+
+def test_reconcile_noop_when_posts_unique(db_sync):
+    a = _make_artist(db_sync, slug="rc2")
+    db_sync.add_all([
+        Post(artist_id=a.id, external_post_id="100", raw_metadata={"post_id": 100}),
+        Post(artist_id=a.id, external_post_id="200", raw_metadata={"post_id": 200}),
+    ])
+    db_sync.commit()
+    assert cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=True)["groups"] == 0
+    res = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
+    assert res["merged"] == 0
+    assert db_sync.execute(select(func.count()).select_from(Post)).scalar_one() == 2
+
+
+def test_reconcile_dedups_provenance_collision(db_sync, tmp_path):
+    # Both dup rows link the SAME image via provenance; merging must DROP the
+    # colliding loser row (unique image_record_id+post_id), not raise.
+    a = _make_artist(db_sync, slug="rc3")
+    img = _make_image(db_sync, artist=a, path=str(tmp_path / "y.jpg"), sha256="d" * 64)
+    legacy = Post(artist_id=a.id, external_post_id="9", raw_metadata={"post_id": 55})
+    native = Post(artist_id=a.id, external_post_id="55", raw_metadata={"post_id": 55})
+    db_sync.add_all([legacy, native])
+    db_sync.flush()
+    db_sync.add(ImageProvenance(image_record_id=img.id, post_id=legacy.id))
+    db_sync.add(ImageProvenance(image_record_id=img.id, post_id=native.id))
+    img_id, native_id = img.id, native.id
+    db_sync.commit()
+
+    cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
+    prov = db_sync.execute(
+        select(ImageProvenance.post_id).where(ImageProvenance.image_record_id == img_id)
+    ).scalars().all()
+    assert prov == [native_id]