feat(maintenance): reconcile duplicate posts (gallery-dl→native unify)
An artist first downloaded by gallery-dl gets Post rows keyed by the per-
attachment id; a later native walk keys the SAME real post by the post id. They
never dedup (uq_post_source_external_id is on external_post_id) → duplicate post
rows (cheunart: 943→1109). The real post id is recoverable in-DB from
raw_metadata['post_id'] (both eras store the sidecar there).
reconcile_duplicate_posts (cleanup_service): group posts by (source_id, canonical
post_id = raw_metadata.post_id else external_post_id); for each group >1, keep the
row already keyed by the post id (the format the CURRENT native downloader
produces, so future walks dedup and this can't recur), re-point
ImageRecord.primary_post_id / ImageProvenance / PostAttachment / ExternalLink onto
it conflict-safe (drop the loser's row where the keeper already has the equivalent,
per each table's uniqueness), backfill the keeper's empty date/title/body/raw_meta
from a loser, set external_post_id=post_id + derive post_url, delete losers.
IMAGES ARE NOT TOUCHED (content-addressed/deduped already; operator-confirmed).
Preview/apply share find_duplicate_post_groups (rule 93). API
/api/admin/posts/reconcile-duplicates (dry_run→{groups,posts_to_merge,sample};
apply→{groups,merged,sample}; optional source_id). UI: a second section on
PostMaintenanceCard (preview groups+sample → confirm merge). Tests: merge +
metadata backfill + image move, no-op when unique, provenance-collision dedup.
Design: milestone #73. Forensics: note #917. Out of scope (flagged): cheunart vs
Cheunart case-variant artist dirs/rows.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -49,12 +49,56 @@
|
||||
class="ml-3 text-caption text-success"
|
||||
>Deleted {{ deleted }} ✓</span>
|
||||
</div>
|
||||
|
||||
<v-divider class="my-5" />
|
||||
|
||||
<p class="fc-muted text-body-2 mb-4">
|
||||
Unify <strong>duplicate posts</strong> — when an artist was first
|
||||
downloaded by gallery-dl and later re-walked by the native ingester, the
|
||||
same post can exist twice (once keyed by the file's attachment id, once by
|
||||
the real post id). This merges each pair onto one canonical post (keeping
|
||||
the native post-id key), moving images, attachments and links onto the
|
||||
survivor. Images themselves are never touched.
|
||||
</p>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingDupPreview"
|
||||
class="mb-3"
|
||||
@click="onPreviewDupes"
|
||||
>Preview duplicate posts</v-btn>
|
||||
|
||||
<div v-if="dupPreview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ dupPreview.groups }}</strong> duplicate group(s),
|
||||
<strong>{{ dupPreview.posts_to_merge }}</strong> redundant row(s) to
|
||||
merge.
|
||||
<span v-if="dupPreview.groups > 50" class="fc-muted">Showing first 50.</span>
|
||||
<span v-else-if="!dupPreview.groups" class="fc-muted">No duplicates found.</span>
|
||||
</p>
|
||||
<SampleNameGrid
|
||||
v-if="dupSampleNames.length"
|
||||
:names="dupSampleNames" class="mb-3"
|
||||
/>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-merge"
|
||||
:disabled="!dupPreview.groups"
|
||||
:loading="merging"
|
||||
@click="onMergeDupes"
|
||||
>Merge {{ dupPreview.posts_to_merge }} duplicate(s)</v-btn>
|
||||
<span
|
||||
v-if="merged != null"
|
||||
class="ml-3 text-caption text-success"
|
||||
>Merged {{ merged }} ✓</span>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useAdminStore } from '../../stores/admin.js'
|
||||
import SampleNameGrid from '../common/SampleNameGrid.vue'
|
||||
@@ -66,6 +110,16 @@ const loadingPreview = ref(false)
|
||||
const committing = ref(false)
|
||||
const deleted = ref(null)
|
||||
|
||||
const dupPreview = ref(null)
|
||||
const loadingDupPreview = ref(false)
|
||||
const merging = ref(false)
|
||||
const merged = ref(null)
|
||||
const dupSampleNames = computed(() =>
|
||||
(dupPreview.value?.sample ?? []).map(
|
||||
(g) => `${g.title} — ${g.rows} rows`,
|
||||
),
|
||||
)
|
||||
|
||||
async function onPreview() {
|
||||
loadingPreview.value = true
|
||||
deleted.value = null
|
||||
@@ -88,6 +142,28 @@ async function onCommit() {
|
||||
committing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onPreviewDupes() {
|
||||
loadingDupPreview.value = true
|
||||
merged.value = null
|
||||
try {
|
||||
dupPreview.value = await store.reconcileDuplicatePosts({ dryRun: true })
|
||||
} finally {
|
||||
loadingDupPreview.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onMergeDupes() {
|
||||
merging.value = true
|
||||
try {
|
||||
const result = await store.reconcileDuplicatePosts({ dryRun: false })
|
||||
merged.value = result.merged ?? 0
|
||||
// Same predicate as the preview — after the merge there are no dup groups left.
|
||||
dupPreview.value = { groups: 0, posts_to_merge: 0, sample: [] }
|
||||
} finally {
|
||||
merging.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -129,6 +129,22 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Tier-A: unify duplicate post rows (gallery-dl attachment-id + native post-id
|
||||
// for the same real post) onto one post-id-keyed keeper. Images untouched.
|
||||
// Preview/apply parity on the backend; UI previews then confirms.
|
||||
async function reconcileDuplicatePosts({ dryRun = true, sourceId = null } = {}) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/posts/reconcile-duplicates',
|
||||
{ body: { dry_run: dryRun, source_id: sourceId } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async function purgeLegacyTags({ dryRun = true } = {}) {
|
||||
lastError.value = null
|
||||
try {
|
||||
@@ -208,6 +224,7 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
tagUsageCount,
|
||||
pruneUnusedTags,
|
||||
pruneBarePosts,
|
||||
reconcileDuplicatePosts,
|
||||
purgeLegacyTags,
|
||||
resetContentTagging,
|
||||
normalizeTags,
|
||||
|
||||
Reference in New Issue
Block a user