feat(cleanup): reclaim orphaned attachments — rows and store blobs (#3068)
PostAttachment's two FKs are both ON DELETE SET NULL, so a deleted post or artist left the row behind rather than taking it. Nothing ever pruned those rows, and nothing in the repo had ever unlinked a file under the attachment store — so both rows and bytes accumulated permanently, invisible to every existing diagnostic. Why a disk->DB reconciliation rather than a row sweep: the store is sha-addressed and idempotent, so ONE blob backs MANY rows. Deleting a row does not free its blob, and since the artist cascade (#3066) now deletes its attachment rows outright, a freed blob has no DB pointer left to find it by. Walking the store and asking "does any row still reference this sha?" catches orphans from every cause, including ones no future delete path will think to report. Preview and apply share `_orphan_attachment_conditions` (rule 93). The dry-run derives its surviving-sha set by NEGATING that same predicate, so it is honest about blobs the delete would free rather than counting them as still-referenced — the one place this was easy to get backwards, so it has its own parity test. Guards, each with a reason: - A blob is written before its row commits, so a just-stored file legitimately has no referencing row. Files under 6h are never judged — same guard and reasoning as ORPHAN_TEMP_MIN_AGE_HOURS. - `.partial` staging files belong to cleanup_orphaned_temp_files; skipped rather than raced. - The sha is parsed as the first 64 chars, not via Path.stem: store() takes the extension from the source filename, and a URL-encoded basename yields a multi-dot suffix that would make stem eat part of the sha. - A 900s walk budget reports partial=True instead of running to the task's hard limit (rule 89). - TASK_STUCK_THRESHOLD_MINUTES override at 30 (= time_limit 25 + 5). Without it a healthy 20-minute walk is phantom-flagged 'RecoverySweep' at the bare 5-min default — the #883 failure class; its invariant test is mirrored here. Defaults to the safe preview at both the task and the route, unlike the other maintenance triggers: this apply unlinks files. Operator-triggered only, never on a beat. Ships with its UI (rule 27): AttachmentReclaimCard in Cleanup → Duplicates & leftovers, built on the existing useMaintenanceTask/MaintenanceTile shapes, so a run survives navigating away. Surfaces files_failed and partial explicitly, since both change what the numbers mean. Also promotes humanBytes to utils/bytes.js — it was byte-identical in VideoDedupCard and GatedPurgeCard and this card would have been the third copy. The three divergent `formatBytes` helpers are deliberately left alone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -459,6 +459,22 @@ async def trigger_prune_missing_files():
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/reclaim-attachments", methods=["POST"])
|
||||
async def trigger_reclaim_attachments():
|
||||
"""Reclaim orphaned attachments (#3068). Body {"dry_run": bool}: dry_run
|
||||
(the DEFAULT here) projects the orphan rows and unreferenced store blobs
|
||||
without touching either; dry_run=false deletes the rows then unlinks every
|
||||
blob no surviving row references. Maintenance queue; operator-triggered
|
||||
only — never an unattended sweep, since the apply unlinks files. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import reclaim_orphaned_attachments_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = reclaim_orphaned_attachments_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
||||
async def trigger_dedup_videos():
|
||||
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
||||
|
||||
@@ -1592,3 +1592,155 @@ def purge_gated_previews(
|
||||
"ledger_cleared": ledger_cleared,
|
||||
"posts_deleted": posts_deleted,
|
||||
}
|
||||
|
||||
|
||||
# -- orphaned attachment reclamation ---------------------------------------
|
||||
# PostAttachment's two FKs are both ON DELETE SET NULL, so a deleted post or
|
||||
# artist leaves the row behind rather than taking it. Nothing ever pruned those
|
||||
# rows, and nothing has ever unlinked a file under the attachment store — so
|
||||
# both rows and bytes accumulated permanently and were invisible to every
|
||||
# existing diagnostic.
|
||||
#
|
||||
# Why this is a DISK->DB reconciliation rather than a row sweep: the store is
|
||||
# sha-addressed and idempotent (attachment_store.store), so ONE blob backs MANY
|
||||
# rows. Deleting a row therefore does not free its blob, and — since the artist
|
||||
# cascade now deletes its attachment rows outright — a freed blob has no DB
|
||||
# pointer left to find it by. Walking the store and asking "does any row still
|
||||
# reference this sha?" catches orphans from every cause, including ones no
|
||||
# future delete path will think to report.
|
||||
|
||||
# A blob is written by attachment_store.store BEFORE its row is inserted and
|
||||
# committed, so a just-stored file legitimately has no referencing row for a
|
||||
# moment. Same guard, same reasoning as ORPHAN_TEMP_MIN_AGE_HOURS in
|
||||
# tasks/maintenance.py: never judge a file younger than this.
|
||||
_ATTACHMENT_ORPHAN_MIN_AGE_HOURS = 6
|
||||
|
||||
# Wall-clock budget for the store walk (rule 89). A library with a large
|
||||
# attachment store shouldn't be able to run this past its soft time limit; on
|
||||
# exhaustion it reports partial=True and the operator re-runs to finish.
|
||||
_ATTACHMENT_RECLAIM_BUDGET_SECONDS = 900
|
||||
|
||||
# The store names files `<sha256><ext>`. Parse the sha as the first 64 chars
|
||||
# rather than via Path.stem: store() takes the extension straight from the
|
||||
# source filename, and a URL-encoded basename yields a multi-dot "suffix"
|
||||
# (see [[reference_url_encoded_basename_suffix]]) that would make stem eat part
|
||||
# of the sha. Validating the 64 chars as hex also skips anything else in the
|
||||
# tree that isn't a stored blob.
|
||||
_SHA256_HEX_LEN = 64
|
||||
|
||||
|
||||
def _orphan_attachment_conditions() -> list:
|
||||
"""PostAttachment rows belonging to nothing: both FKs nulled by a deleted
|
||||
post AND a deleted artist. A row with post_id NULL but an artist_id is the
|
||||
deliberate filesystem-import case (importer._capture_attachment writes it
|
||||
that way) and is NOT an orphan — it is still attributed."""
|
||||
return [
|
||||
PostAttachment.post_id.is_(None),
|
||||
PostAttachment.artist_id.is_(None),
|
||||
]
|
||||
|
||||
|
||||
def _is_sha_named(name: str) -> bool:
|
||||
"""True when `name` starts with a 64-char lowercase-hex sha256."""
|
||||
if len(name) < _SHA256_HEX_LEN:
|
||||
return False
|
||||
head = name[:_SHA256_HEX_LEN]
|
||||
return all(c in "0123456789abcdef" for c in head)
|
||||
|
||||
|
||||
def reclaim_orphaned_attachments(
|
||||
session: Session, *, images_root: Path, dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Prune unattributed PostAttachment rows, then unlink store blobs that no
|
||||
surviving row references.
|
||||
|
||||
Returns (same discovery keys either way, so the UI renders one shape):
|
||||
{"rows": int, # orphan rows found / deleted
|
||||
"files": int, # unreferenced blobs found / unlinked
|
||||
"bytes": int, # their total size
|
||||
"scanned": int, # blobs examined
|
||||
"skipped_recent": int, # blobs under the min-age guard
|
||||
"files_failed": int, # unlink raised (apply only)
|
||||
"partial": bool} # walk hit the time budget
|
||||
|
||||
dry_run computes exactly what the apply would do and mutates nothing — the
|
||||
surviving-sha set is derived by NEGATING the same orphan predicate the
|
||||
delete uses, so the preview cannot disagree with the apply (rule 93).
|
||||
"""
|
||||
started = time.monotonic()
|
||||
orphan_conds = _orphan_attachment_conditions()
|
||||
|
||||
if dry_run:
|
||||
rows = session.execute(
|
||||
select(func.count(PostAttachment.id)).where(*orphan_conds)
|
||||
).scalar_one()
|
||||
else:
|
||||
rows = session.execute(
|
||||
delete(PostAttachment).where(*orphan_conds)
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
|
||||
# Shas that still have a home. In the apply path the orphan rows are already
|
||||
# gone, so `NOT orphan` is redundant but harmless; in the dry-run path it is
|
||||
# what makes the projection honest about blobs the delete would free. One
|
||||
# predicate, one query, both modes.
|
||||
surviving_shas = set(session.execute(
|
||||
select(PostAttachment.sha256).where(~and_(*orphan_conds)).distinct()
|
||||
).scalars())
|
||||
|
||||
root = Path(images_root) / "attachments"
|
||||
cutoff = (
|
||||
datetime.now(UTC).timestamp()
|
||||
- _ATTACHMENT_ORPHAN_MIN_AGE_HOURS * 3600
|
||||
)
|
||||
files = 0
|
||||
freed_bytes = 0
|
||||
scanned = 0
|
||||
skipped_recent = 0
|
||||
files_failed = 0
|
||||
partial = False
|
||||
|
||||
if root.is_dir():
|
||||
for path in root.rglob("*"):
|
||||
if time.monotonic() - started >= _ATTACHMENT_RECLAIM_BUDGET_SECONDS:
|
||||
partial = True
|
||||
break
|
||||
# .partial staging files belong to cleanup_orphaned_temp_files —
|
||||
# leave them alone rather than racing an in-flight store().
|
||||
if path.suffix in (".part", ".partial") or not path.is_file():
|
||||
continue
|
||||
if not _is_sha_named(path.name):
|
||||
continue
|
||||
scanned += 1
|
||||
sha = path.name[:_SHA256_HEX_LEN]
|
||||
if sha in surviving_shas:
|
||||
continue
|
||||
try:
|
||||
st = path.stat()
|
||||
if st.st_mtime >= cutoff:
|
||||
skipped_recent += 1
|
||||
continue
|
||||
size = st.st_size
|
||||
if not dry_run:
|
||||
path.unlink()
|
||||
files += 1
|
||||
freed_bytes += size
|
||||
except OSError as exc:
|
||||
files_failed += 1
|
||||
log.warning("reclaim_orphaned_attachments: %s: %s", path, exc)
|
||||
|
||||
if not dry_run and (rows or files):
|
||||
log.info(
|
||||
"attachment reclaim: %d orphan row(s) deleted, %d blob(s) unlinked "
|
||||
"(%d bytes), %d failed, partial=%s",
|
||||
rows, files, freed_bytes, files_failed, partial,
|
||||
)
|
||||
return {
|
||||
"rows": rows,
|
||||
"files": files,
|
||||
"bytes": freed_bytes,
|
||||
"scanned": scanned,
|
||||
"skipped_recent": skipped_recent,
|
||||
"files_failed": files_failed,
|
||||
"partial": partial,
|
||||
}
|
||||
|
||||
@@ -409,3 +409,31 @@ def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict:
|
||||
)
|
||||
rescan_series_suggestions_task.delay(summary["resume_after_id"])
|
||||
return summary
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.reclaim_orphaned_attachments_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
# The service stops walking at its own 900s budget and reports partial, so
|
||||
# these limits are the backstop for a wedged filesystem (NFS stall), not the
|
||||
# expected exit. Comfortably above the budget so a normal run always returns
|
||||
# its summary rather than being killed mid-walk.
|
||||
soft_time_limit=1200, time_limit=1500, # 20 min / 25 min
|
||||
)
|
||||
def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
|
||||
"""Reclaim unattributed PostAttachment rows and the store blobs nothing
|
||||
references any more (#3068). dry_run (the default) returns the projection
|
||||
without touching rows or files; apply deletes the orphan rows, then unlinks
|
||||
every blob no surviving row references.
|
||||
|
||||
Defaults to the SAFE preview — unlike the other tasks here, whose apply is
|
||||
reversible-ish or scoped; this one deletes files. Operator-triggered only,
|
||||
never on a beat: an unattended sweep that unlinks blobs is not something to
|
||||
run without someone reading the projection first."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return cleanup_service.reclaim_orphaned_attachments(
|
||||
session, images_root=IMAGES_ROOT, dry_run=dry_run,
|
||||
)
|
||||
|
||||
@@ -173,6 +173,12 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# task-name override beats the queue threshold whatever queue the row records
|
||||
# (it recorded 'default' before the celery_signals fix → download). 65 = 60+5.
|
||||
"backend.app.tasks.external.fetch_external_link": 65,
|
||||
# Attachment reclaim walks the whole sha-addressed store; the service caps
|
||||
# itself at a 900s budget and reports partial, but the task's hard limit is
|
||||
# 25 min for a wedged filesystem (NFS stall). Same phantom-flag class as the
|
||||
# external-fetch entry above — without an override a healthy in-flight walk
|
||||
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
<template>
|
||||
<!-- #3068: attachment reclamation. PostAttachment's FKs are both SET NULL, so
|
||||
a deleted post or artist leaves the row behind; and the store is
|
||||
sha-addressed, so one blob backs many rows and deleting a row never freed
|
||||
its file. Nothing swept either. Preview first, then apply (destructive:
|
||||
unlinks files). -->
|
||||
<MaintenanceTile
|
||||
icon="mdi-paperclip-off"
|
||||
title="Reclaim orphaned attachments"
|
||||
blurb="Remove attachment records belonging to nothing, and the files nothing references."
|
||||
destructive
|
||||
:open="applying || previewing"
|
||||
>
|
||||
<p class="text-body-2 mb-3">
|
||||
Attachment records survive the post and artist they belonged to, and the
|
||||
files behind them are shared between records — so a deleted record never
|
||||
freed its file on its own. This finds records attributed to
|
||||
<strong>neither</strong> a post nor an artist, and files in the attachment
|
||||
store that <strong>no remaining record</strong> points at.
|
||||
<strong>Preview</strong> first; <strong>Apply</strong> deletes those
|
||||
records and unlinks those files. Files written in the last few hours are
|
||||
always left alone, so an in-progress download is never caught mid-write.
|
||||
</p>
|
||||
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" variant="tonal" rounded="pill"
|
||||
:loading="previewing" :disabled="applying" @click="preview"
|
||||
>
|
||||
<v-icon start>mdi-magnify</v-icon> Preview
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="error" rounded="pill"
|
||||
:loading="applying"
|
||||
:disabled="previewing || !canApply"
|
||||
@click="confirmOpen = true"
|
||||
>
|
||||
<v-icon start>mdi-paperclip-off</v-icon> Apply
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||
density="comfortable"
|
||||
>
|
||||
<span v-if="applied">
|
||||
Deleted {{ summary.rows }} orphaned record(s) and unlinked
|
||||
{{ summary.files }} file(s), reclaiming {{ humanBytes(summary.bytes) }}.
|
||||
</span>
|
||||
<span v-else-if="hasWork">
|
||||
{{ summary.rows }} orphaned record(s) and {{ summary.files }}
|
||||
unreferenced file(s) — {{ humanBytes(summary.bytes) }} reclaimable.
|
||||
Click <strong>Apply</strong> to remove them.
|
||||
</span>
|
||||
<span v-else>Nothing to reclaim — every attachment is accounted for.</span>
|
||||
|
||||
<!-- Both of these change what the numbers MEAN, so they are stated
|
||||
whenever they are non-zero rather than hidden in a tooltip. -->
|
||||
<div v-if="summary.files_failed" class="mt-1 text-caption">
|
||||
{{ summary.files_failed }} file(s) could not be read or removed — see
|
||||
the worker log.
|
||||
</div>
|
||||
<div v-if="summary.partial" class="mt-1 text-caption">
|
||||
Stopped early at the time limit; some of the store was not examined.
|
||||
Run it again to continue.
|
||||
</div>
|
||||
</v-alert>
|
||||
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Maintenance" />
|
||||
|
||||
<v-dialog v-model="confirmOpen" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title>Reclaim orphaned attachments?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
This permanently deletes
|
||||
<strong>{{ summary?.rows ?? 0 }}</strong> attachment record(s) and
|
||||
unlinks <strong>{{ summary?.files ?? 0 }}</strong> file(s)
|
||||
({{ humanBytes(summary?.bytes) }}). Only files that no remaining
|
||||
record points at are removed, so nothing still attached to a post
|
||||
is affected.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||
<v-btn color="error" @click="apply">Reclaim</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
// Walks the whole attachment store, so it can run for minutes on a large
|
||||
// library — the service caps itself at 900s and reports `partial`. 150 polls
|
||||
// × 2s ≈ 5m of foreground waiting; past that the composable hands off to the
|
||||
// task dashboard rather than spinning forever.
|
||||
const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
|
||||
endpoint: '/api/admin/maintenance/reclaim-attachments',
|
||||
storageKey: 'fc.maint.reclaimAttachments',
|
||||
appliedToast: 'Orphaned attachments reclaimed',
|
||||
maxPolls: 150,
|
||||
})
|
||||
|
||||
const hasWork = computed(
|
||||
() => !!summary.value && (summary.value.rows > 0 || summary.value.files > 0),
|
||||
)
|
||||
const canApply = computed(() => hasWork.value && !applied.value)
|
||||
const summaryType = computed(() => {
|
||||
if (applied.value) return 'success'
|
||||
return hasWork.value ? 'info' : 'success'
|
||||
})
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
applyTask()
|
||||
}
|
||||
</script>
|
||||
@@ -102,6 +102,7 @@
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
@@ -122,14 +123,6 @@ const summaryType = computed(() => {
|
||||
return summary.value && summary.value.matched > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
|
||||
@@ -78,6 +78,7 @@
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
@@ -98,14 +99,6 @@ const summaryType = computed(() => {
|
||||
return summary.value && summary.value.redundant > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Human-readable byte sizes for maintenance summaries ("2.4 GB reclaimable").
|
||||
//
|
||||
// Promoted out of the cleanup cards, which had grown byte-identical private
|
||||
// copies (VideoDedupCard, GatedPurgeCard) and were about to grow a third for
|
||||
// the attachment reclaim. Binary units (1 KB = 1024 B) — these numbers come
|
||||
// from st_size / SUM(size_bytes), so they describe disk, not marketing.
|
||||
//
|
||||
// NOT the same shape as the `formatBytes` helpers in SystemStatsCards,
|
||||
// BackupRunsTable and PostCard — those differ in units, precision and
|
||||
// zero-handling. Left alone deliberately rather than force-fitted here.
|
||||
export function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
@@ -19,14 +19,16 @@
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Duplicates & posts</h3>
|
||||
<h3 class="fc-section__title">Duplicates & leftovers</h3>
|
||||
<p class="fc-section__hint">
|
||||
Tidy post records, duplicates and locked-preview leftovers.
|
||||
Tidy post records, duplicates, locked-preview leftovers and attachments
|
||||
that outlived what they belonged to.
|
||||
</p>
|
||||
<div class="fc-tile-grid">
|
||||
<PostMaintenanceCard />
|
||||
<VideoDedupCard />
|
||||
<GatedPurgeCard />
|
||||
<AttachmentReclaimCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -60,6 +62,7 @@ import SingleColorAuditCard from '../components/cleanup/SingleColorAuditCard.vue
|
||||
import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
|
||||
import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
|
||||
import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
|
||||
import AttachmentReclaimCard from '../components/settings/AttachmentReclaimCard.vue'
|
||||
import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
|
||||
import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
|
||||
</script>
|
||||
|
||||
@@ -677,3 +677,34 @@ async def test_reset_content_tagging_apply_requires_confirm_token(client, db):
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert (await resp.get_json())["deleted"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_reclaim_attachments_defaults_to_preview(client, monkeypatch):
|
||||
"""Unlike the other maintenance triggers, this one's apply unlinks FILES —
|
||||
so an empty body must mean preview, not apply."""
|
||||
from backend.app.tasks import admin as admin_tasks
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
admin_tasks.reclaim_orphaned_attachments_task, "delay", _fake_delay(calls)
|
||||
)
|
||||
resp = await client.post("/api/admin/maintenance/reclaim-attachments", json={})
|
||||
assert resp.status_code == 202
|
||||
assert (await resp.get_json())["task_id"] == "task-xyz"
|
||||
assert calls[0][1] == {"dry_run": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_reclaim_attachments_threads_apply(client, monkeypatch):
|
||||
from backend.app.tasks import admin as admin_tasks
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
admin_tasks.reclaim_orphaned_attachments_task, "delay", _fake_delay(calls)
|
||||
)
|
||||
resp = await client.post(
|
||||
"/api/admin/maintenance/reclaim-attachments", json={"dry_run": False},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
assert calls[0][1] == {"dry_run": False}
|
||||
|
||||
@@ -5,6 +5,7 @@ side effects use tmp_path. Assertions on mutated rows use COLUMN
|
||||
SELECTS per reference_async_coredml_test_assertions — never
|
||||
re-read ORM attributes after a service mutates and re-fetches.
|
||||
"""
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
@@ -1081,3 +1082,176 @@ def test_reconcile_preserves_from_attachment_on_provenance_collision(db_sync, tm
|
||||
.where(ImageProvenance.image_record_id == img_id)
|
||||
).all()
|
||||
assert rows == [(native_id, att_id)]
|
||||
|
||||
|
||||
# --- reclaim_orphaned_attachments -----------------------------------
|
||||
|
||||
|
||||
def _store_blob(root, sha, *, ext=".pdf", age_hours=48, data=b"blob"):
|
||||
"""Write a file into the sha-addressed attachment store, aged past the
|
||||
min-age guard by default."""
|
||||
d = root / "attachments" / sha[:3]
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
p = d / f"{sha}{ext}"
|
||||
p.write_bytes(data)
|
||||
old = datetime.now(UTC).timestamp() - age_hours * 3600
|
||||
os.utime(p, (old, old))
|
||||
return p
|
||||
|
||||
|
||||
def _attachment(db_sync, *, sha, post=None, artist=None):
|
||||
att = PostAttachment(
|
||||
post_id=post.id if post else None,
|
||||
artist_id=artist.id if artist else None,
|
||||
sha256=sha, path=f"/store/{sha[:3]}/f.pdf",
|
||||
original_filename="f.pdf", ext=".pdf", size_bytes=4,
|
||||
)
|
||||
db_sync.add(att)
|
||||
db_sync.flush()
|
||||
return att
|
||||
|
||||
|
||||
def test_reclaim_attachments_dry_run_projects_without_mutating(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="recl-dry")
|
||||
p = Post(artist_id=a.id, external_post_id="rd-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
kept_sha, orphan_sha = "aa11".ljust(64, "0"), "bb22".ljust(64, "0")
|
||||
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
||||
_attachment(db_sync, sha=orphan_sha) # both FKs NULL → orphan
|
||||
db_sync.commit()
|
||||
kept_blob = _store_blob(tmp_path, kept_sha)
|
||||
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
assert result["rows"] == 1
|
||||
assert result["files"] == 1
|
||||
assert result["bytes"] == orphan_blob.stat().st_size
|
||||
|
||||
# Nothing actually happened.
|
||||
assert kept_blob.exists() and orphan_blob.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count(PostAttachment.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
|
||||
def test_reclaim_attachments_apply_deletes_rows_and_unlinks_blobs(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="recl-apply")
|
||||
p = Post(artist_id=a.id, external_post_id="ra-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
kept_sha, orphan_sha = "cc33".ljust(64, "0"), "dd44".ljust(64, "0")
|
||||
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
||||
_attachment(db_sync, sha=orphan_sha)
|
||||
db_sync.commit()
|
||||
kept_blob = _store_blob(tmp_path, kept_sha)
|
||||
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 1
|
||||
assert result["files"] == 1
|
||||
|
||||
assert kept_blob.exists() # still referenced
|
||||
assert not orphan_blob.exists() # nothing points at it any more
|
||||
surviving = db_sync.execute(select(PostAttachment.sha256)).scalars().all()
|
||||
assert surviving == [kept_sha]
|
||||
|
||||
|
||||
def test_reclaim_attachments_preview_matches_apply(db_sync, tmp_path):
|
||||
"""Rule 93 — the dry-run's numbers are what the apply does. The projection
|
||||
has to negate the orphan predicate to be honest about blobs the delete is
|
||||
about to free, so this is the assertion that catches getting that backwards.
|
||||
"""
|
||||
orphan_sha = "ee55".ljust(64, "0")
|
||||
_attachment(db_sync, sha=orphan_sha)
|
||||
db_sync.commit()
|
||||
_store_blob(tmp_path, orphan_sha)
|
||||
|
||||
projected = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
applied = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
for key in ("rows", "files", "bytes"):
|
||||
assert projected[key] == applied[key], key
|
||||
assert applied["rows"] == 1 and applied["files"] == 1
|
||||
|
||||
|
||||
def test_reclaim_attachments_keeps_shared_blob_while_any_row_remains(db_sync, tmp_path):
|
||||
"""The refcount case this whole sweep exists for: one sha-addressed blob
|
||||
backs several rows, so deleting SOME of them must not free the file."""
|
||||
a = _make_artist(db_sync, slug="recl-shared")
|
||||
p = Post(artist_id=a.id, external_post_id="rs-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
sha = "ff66".ljust(64, "0")
|
||||
_attachment(db_sync, sha=sha, post=p, artist=a) # attributed — survives
|
||||
_attachment(db_sync, sha=sha) # orphan — deleted
|
||||
db_sync.commit()
|
||||
blob = _store_blob(tmp_path, sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 1 # the orphan row went
|
||||
assert result["files"] == 0 # the blob did NOT
|
||||
assert blob.exists()
|
||||
|
||||
|
||||
def test_reclaim_attachments_spares_filesystem_import_rows(db_sync, tmp_path):
|
||||
"""post_id NULL with an artist_id is the deliberate filesystem-import shape
|
||||
(importer._capture_attachment), not an orphan — it is still attributed."""
|
||||
a = _make_artist(db_sync, slug="recl-fsimport")
|
||||
sha = "1177".ljust(64, "0")
|
||||
_attachment(db_sync, sha=sha, artist=a) # post NULL, artist set
|
||||
db_sync.commit()
|
||||
blob = _store_blob(tmp_path, sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 0
|
||||
assert result["files"] == 0
|
||||
assert blob.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count(PostAttachment.id))
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
def test_reclaim_attachments_skips_recent_and_staging_files(db_sync, tmp_path):
|
||||
"""A blob is written BEFORE its row commits, so a just-stored file with no
|
||||
row is in-flight, not orphaned. `.partial` staging files belong to
|
||||
cleanup_orphaned_temp_files and must be left alone either way."""
|
||||
fresh_sha, staged_sha = "2288".ljust(64, "0"), "3399".ljust(64, "0")
|
||||
fresh = _store_blob(tmp_path, fresh_sha, age_hours=0)
|
||||
staged = _store_blob(tmp_path, staged_sha, ext=".pdf.partial")
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["files"] == 0
|
||||
assert result["skipped_recent"] == 1
|
||||
assert fresh.exists() and staged.exists()
|
||||
|
||||
|
||||
def test_reclaim_attachments_ignores_non_sha_named_files(db_sync, tmp_path):
|
||||
"""The walk must only judge files it can identify as store blobs — anything
|
||||
else under the root is none of its business."""
|
||||
d = tmp_path / "attachments" / "zzz"
|
||||
d.mkdir(parents=True)
|
||||
stray = d / "notes.txt"
|
||||
stray.write_text("not a blob")
|
||||
old = datetime.now(UTC).timestamp() - 48 * 3600
|
||||
os.utime(stray, (old, old))
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["files"] == 0
|
||||
assert stray.exists()
|
||||
|
||||
@@ -778,3 +778,17 @@ def test_vacuum_analyze_runs_over_high_churn_tables():
|
||||
|
||||
result = vacuum_analyze.apply().get()
|
||||
assert result["vacuumed"] == list(VACUUM_TABLES)
|
||||
|
||||
|
||||
def test_reclaim_attachments_stuck_threshold_exceeds_hard_time_limit():
|
||||
"""#883's invariant, applied to the attachment reclaim: a task whose stall
|
||||
threshold is under its own hard limit gets phantom-flagged 'RecoverySweep'
|
||||
while it is still healthily running."""
|
||||
from backend.app.tasks.admin import reclaim_orphaned_attachments_task
|
||||
from backend.app.tasks.maintenance import TASK_STUCK_THRESHOLD_MINUTES
|
||||
|
||||
hard_minutes = reclaim_orphaned_attachments_task.time_limit / 60
|
||||
override = TASK_STUCK_THRESHOLD_MINUTES[
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task"
|
||||
]
|
||||
assert override >= hard_minutes
|
||||
|
||||
Reference in New Issue
Block a user