From eff64275fc4b58a50171b57c3e27426faa683ba4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 21:57:21 -0400 Subject: [PATCH] =?UTF-8?q?feat(maintenance):=20reconcile=20duplicate=20po?= =?UTF-8?q?sts=20(gallery-dl=E2=86=92native=20unify)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An artist first downloaded by gallery-dl gets Post rows keyed by the per- attachment id; a later native walk keys the SAME real post by the post id. They never dedup (uq_post_source_external_id is on external_post_id) → duplicate post rows (cheunart: 943→1109). The real post id is recoverable in-DB from raw_metadata['post_id'] (both eras store the sidecar there). reconcile_duplicate_posts (cleanup_service): group posts by (source_id, canonical post_id = raw_metadata.post_id else external_post_id); for each group >1, keep the row already keyed by the post id (the format the CURRENT native downloader produces, so future walks dedup and this can't recur), re-point ImageRecord.primary_post_id / ImageProvenance / PostAttachment / ExternalLink onto it conflict-safe (drop the loser's row where the keeper already has the equivalent, per each table's uniqueness), backfill the keeper's empty date/title/body/raw_meta from a loser, set external_post_id=post_id + derive post_url, delete losers. IMAGES ARE NOT TOUCHED (content-addressed/deduped already; operator-confirmed). Preview/apply share find_duplicate_post_groups (rule 93). API /api/admin/posts/reconcile-duplicates (dry_run→{groups,posts_to_merge,sample}; apply→{groups,merged,sample}; optional source_id). UI: a second section on PostMaintenanceCard (preview groups+sample → confirm merge). Tests: merge + metadata backfill + image move, no-op when unique, provenance-collision dedup. Design: milestone #73. Forensics: note #917. Out of scope (flagged): cheunart vs Cheunart case-variant artist dirs/rows. Co-Authored-By: Claude Opus 4.8 --- backend/app/api/admin.py | 27 +++ backend/app/services/cleanup_service.py | 178 ++++++++++++++++++ .../settings/PostMaintenanceCard.vue | 78 +++++++- frontend/src/stores/admin.js | 17 ++ tests/test_cleanup_service.py | 86 +++++++++ 5 files changed, 385 insertions(+), 1 deletion(-) diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index efd6468..10bf684 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -226,6 +226,33 @@ async def posts_prune_bare(): return jsonify(result) +@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"]) +async def posts_reconcile_duplicates(): + """Tier-A: unify duplicate post rows for the same real post — the gallery-dl + (attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper, + moving image/provenance/attachment/link rows over. Images are untouched. + dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies + and returns {groups, merged, sample}. Optional source_id scopes to one source. + Same find_duplicate_post_groups predicate drives preview + apply (rule 93).""" + from ..services.cleanup_service import reconcile_duplicate_posts + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", False)) + raw_source = body.get("source_id") + try: + source_id = int(raw_source) if raw_source is not None else None + except (TypeError, ValueError): + return _bad("invalid_source_id", detail="source_id must be an integer") + + async with get_session() as session: + result = await session.run_sync( + lambda sync_sess: reconcile_duplicate_posts( + sync_sess, source_id=source_id, dry_run=dry_run, + ) + ) + return jsonify(result) + + @admin_bp.route("/tags/purge-legacy", methods=["POST"]) async def tags_purge_legacy(): """Tier-A: delete legacy IR-migration tags — archive/post/artist diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index ab9ad8d..9979907 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -22,6 +22,7 @@ from sqlalchemy.orm import Session from ..models import ( Artist, + ExternalLink, ImageProvenance, ImageRecord, LibraryAuditRun, @@ -36,6 +37,7 @@ from ..models.series_page import SeriesPage from ..models.tag import image_tag from ..utils import safe_probe from .importer import _VIDEO_DUP_ASPECT_TOL, _VIDEO_DUP_DURATION_TOL_SECONDS +from .platforms import PLATFORMS log = logging.getLogger(__name__) @@ -507,6 +509,182 @@ def prune_bare_posts(session: Session, *, dry_run: bool = False) -> dict: return {"deleted": result.rowcount or 0, "sample_names": sample} +# -- duplicate-post reconciliation (gallery-dl → native migration) ---------- +# An artist first downloaded by gallery-dl gets Post rows keyed by the per- +# ATTACHMENT id (gallery-dl's `id`); a later native walk keys the SAME real post +# by the post id. The two never dedup (uq_post_source_external_id is on +# external_post_id) → duplicate post rows. The real post id is recoverable in-DB +# from raw_metadata["post_id"] (both eras store the sidecar there). We unify each +# group onto ONE post row keyed the way the CURRENT native downloader keys it +# (post id), so future native walks match and the dup can't recur. Images are +# untouched (content-addressed/deduped already); only post rows + their link +# rows move. Milestone #73 / note #917. + + +def _canonical_post_id(post: Post) -> str | None: + """The real platform post id used to group duplicate rows: raw_metadata + ['post_id'] when present (the true id, stored by both gallery-dl and native + imports), else external_post_id. None when neither is usable.""" + rm = post.raw_metadata or {} + pid = rm.get("post_id") + if pid is not None and str(pid).strip(): + return str(pid) + return post.external_post_id or None + + +def find_duplicate_post_groups( + session: Session, *, source_id: int | None = None, +) -> list[list[Post]]: + """Groups of >1 Post that are the SAME real post (same source_id + canonical + post id) — the gallery-dl(attachment-id) / native(post-id) duplicates. Shared + by the dry-run preview and the live reconcile (preview/apply parity).""" + stmt = select(Post) + if source_id is not None: + stmt = stmt.where(Post.source_id == source_id) + groups: dict[tuple, list[Post]] = {} + for post in session.execute(stmt.order_by(Post.id)).scalars().all(): + cpid = _canonical_post_id(post) + if not cpid: + continue + groups.setdefault((post.source_id, cpid), []).append(post) + return [posts for posts in groups.values() if len(posts) > 1] + + +def _choose_keeper(posts: list[Post], cpid: str) -> Post: + """The surviving row for a dup group: prefer one already keyed by the + canonical post id (the native format we keep), then the most complete + (has description, has date), then the lowest id for stability.""" + native = [p for p in posts if p.external_post_id == cpid] + pool = native or posts + return min(pool, key=lambda p: (not p.description, not p.post_date, p.id)) + + +def _repoint_post_links(session: Session, loser_id: int, keeper_id: int) -> None: + """Move every link row from a loser post to the keeper, conflict-safe against + each table's uniqueness (drop the loser's row when the keeper already has the + equivalent, else re-point). Images themselves are never touched.""" + # ImageRecord.primary_post_id — no uniqueness; straight re-point. + session.execute( + update(ImageRecord) + .where(ImageRecord.primary_post_id == loser_id) + .values(primary_post_id=keeper_id) + ) + # ImageProvenance — unique (image_record_id, post_id). + dup_imgs = select(ImageProvenance.image_record_id).where( + ImageProvenance.post_id == keeper_id + ) + session.execute( + delete(ImageProvenance).where( + ImageProvenance.post_id == loser_id, + ImageProvenance.image_record_id.in_(dup_imgs), + ) + ) + session.execute( + update(ImageProvenance) + .where(ImageProvenance.post_id == loser_id) + .values(post_id=keeper_id) + ) + # PostAttachment — partial unique (post_id, sha256) where post_id NOT NULL. + dup_shas = select(PostAttachment.sha256).where(PostAttachment.post_id == keeper_id) + session.execute( + delete(PostAttachment).where( + PostAttachment.post_id == loser_id, + PostAttachment.sha256.in_(dup_shas), + ) + ) + session.execute( + update(PostAttachment) + .where(PostAttachment.post_id == loser_id) + .values(post_id=keeper_id) + ) + # ExternalLink — unique (post_id, url). + dup_urls = select(ExternalLink.url).where(ExternalLink.post_id == keeper_id) + session.execute( + delete(ExternalLink).where( + ExternalLink.post_id == loser_id, + ExternalLink.url.in_(dup_urls), + ) + ) + session.execute( + update(ExternalLink) + .where(ExternalLink.post_id == loser_id) + .values(post_id=keeper_id) + ) + + +def _fill_missing_post_fields(keeper: Post, loser: Post) -> None: + """Backfill the keeper's empty metadata from a loser that has it (the native + stub is often bare; the gallery-dl row carries date/title/body/raw_metadata).""" + if not keeper.post_date and loser.post_date: + keeper.post_date = loser.post_date + if not keeper.post_title and loser.post_title: + keeper.post_title = loser.post_title + if not keeper.description and loser.description: + keeper.description = loser.description + if not keeper.raw_metadata and loser.raw_metadata: + keeper.raw_metadata = loser.raw_metadata + if not keeper.attachment_count and loser.attachment_count: + keeper.attachment_count = loser.attachment_count + + +def _canonical_post_url(post: Post, cpid: str) -> str | None: + """Permalink for the unified post, via the platform's derive_post_url hook + (subscribestar/patreon synthesize `…/posts/`). Falls back to the keeper's + existing url when no hook applies.""" + platform = (post.raw_metadata or {}).get("category") + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.derive_post_url is not None: + derived = info.derive_post_url({"post_id": cpid}) + if derived: + return derived + return post.post_url + + +def reconcile_duplicate_posts( + session: Session, *, source_id: int | None = None, dry_run: bool = False, +) -> dict: + """Unify duplicate post rows (gallery-dl attachment-id + native post-id) onto + one keeper per real post, re-keyed to the post id. Images untouched. + + Returns: + dry_run=True: {"groups": G, "posts_to_merge": L, "sample": [...]} + dry_run=False: {"groups": G, "merged": L, "sample": [...]} + where L = rows that would be (were) deleted after merging into keepers. The + SAME find_duplicate_post_groups predicate drives preview and apply (rule 93). + """ + groups = find_duplicate_post_groups(session, source_id=source_id) + sample: list[dict] = [] + losers_total = 0 + for posts in groups: + cpid = _canonical_post_id(posts[0]) + keeper = _choose_keeper(posts, cpid) + losers = [p for p in posts if p.id != keeper.id] + losers_total += len(losers) + if len(sample) < 50: + sample.append({ + "post_id": cpid, + "rows": len(posts), + "keeper_id": keeper.id, + "title": keeper.post_title or f"Post {cpid}", + }) + if dry_run: + continue + for loser in losers: + _repoint_post_links(session, loser.id, keeper.id) + _fill_missing_post_fields(keeper, loser) + keeper.external_post_id = cpid + new_url = _canonical_post_url(keeper, cpid) + if new_url: + keeper.post_url = new_url + session.flush() + for loser in losers: + session.delete(loser) + if dry_run: + return {"groups": len(groups), "posts_to_merge": losers_total, "sample": sample} + session.commit() + return {"groups": len(groups), "merged": losers_total, "sample": sample} + + # Legacy tags FC no longer uses, in two shapes: # (1) kinds the tag input never produces — archive/post/artist. # provenance (post grouping) + archive membership are their own diff --git a/frontend/src/components/settings/PostMaintenanceCard.vue b/frontend/src/components/settings/PostMaintenanceCard.vue index 71b07cd..ee54039 100644 --- a/frontend/src/components/settings/PostMaintenanceCard.vue +++ b/frontend/src/components/settings/PostMaintenanceCard.vue @@ -49,12 +49,56 @@ class="ml-3 text-caption text-success" >Deleted {{ deleted }} ✓ + + + +

+ Unify duplicate posts — when an artist was first + downloaded by gallery-dl and later re-walked by the native ingester, the + same post can exist twice (once keyed by the file's attachment id, once by + the real post id). This merges each pair onto one canonical post (keeping + the native post-id key), moving images, attachments and links onto the + survivor. Images themselves are never touched. +

+ + Preview duplicate posts + +
+

+ {{ dupPreview.groups }} duplicate group(s), + {{ dupPreview.posts_to_merge }} redundant row(s) to + merge. + Showing first 50. + No duplicates found. +

+ + Merge {{ dupPreview.posts_to_merge }} duplicate(s) + Merged {{ merged }} ✓ +