Extension channels: dev and main each carry their own signed extension #237
@@ -459,6 +459,22 @@ async def trigger_prune_missing_files():
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/reclaim-attachments", methods=["POST"])
|
||||
async def trigger_reclaim_attachments():
|
||||
"""Reclaim orphaned attachments (#3068). Body {"dry_run": bool}: dry_run
|
||||
(the DEFAULT here) projects the orphan rows and unreferenced store blobs
|
||||
without touching either; dry_run=false deletes the rows then unlinks every
|
||||
blob no surviving row references. Maintenance queue; operator-triggered
|
||||
only — never an unattended sweep, since the apply unlinks files. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import reclaim_orphaned_attachments_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = reclaim_orphaned_attachments_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
||||
async def trigger_dedup_videos():
|
||||
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
||||
|
||||
@@ -1592,3 +1592,155 @@ def purge_gated_previews(
|
||||
"ledger_cleared": ledger_cleared,
|
||||
"posts_deleted": posts_deleted,
|
||||
}
|
||||
|
||||
|
||||
# -- orphaned attachment reclamation ---------------------------------------
|
||||
# PostAttachment's two FKs are both ON DELETE SET NULL, so a deleted post or
|
||||
# artist leaves the row behind rather than taking it. Nothing ever pruned those
|
||||
# rows, and nothing has ever unlinked a file under the attachment store — so
|
||||
# both rows and bytes accumulated permanently and were invisible to every
|
||||
# existing diagnostic.
|
||||
#
|
||||
# Why this is a DISK->DB reconciliation rather than a row sweep: the store is
|
||||
# sha-addressed and idempotent (attachment_store.store), so ONE blob backs MANY
|
||||
# rows. Deleting a row therefore does not free its blob, and — since the artist
|
||||
# cascade now deletes its attachment rows outright — a freed blob has no DB
|
||||
# pointer left to find it by. Walking the store and asking "does any row still
|
||||
# reference this sha?" catches orphans from every cause, including ones no
|
||||
# future delete path will think to report.
|
||||
|
||||
# A blob is written by attachment_store.store BEFORE its row is inserted and
|
||||
# committed, so a just-stored file legitimately has no referencing row for a
|
||||
# moment. Same guard, same reasoning as ORPHAN_TEMP_MIN_AGE_HOURS in
|
||||
# tasks/maintenance.py: never judge a file younger than this.
|
||||
_ATTACHMENT_ORPHAN_MIN_AGE_HOURS = 6
|
||||
|
||||
# Wall-clock budget for the store walk (rule 89). A library with a large
|
||||
# attachment store shouldn't be able to run this past its soft time limit; on
|
||||
# exhaustion it reports partial=True and the operator re-runs to finish.
|
||||
_ATTACHMENT_RECLAIM_BUDGET_SECONDS = 900
|
||||
|
||||
# The store names files `<sha256><ext>`. Parse the sha as the first 64 chars
|
||||
# rather than via Path.stem: store() takes the extension straight from the
|
||||
# source filename, and a URL-encoded basename yields a multi-dot "suffix"
|
||||
# (see [[reference_url_encoded_basename_suffix]]) that would make stem eat part
|
||||
# of the sha. Validating the 64 chars as hex also skips anything else in the
|
||||
# tree that isn't a stored blob.
|
||||
_SHA256_HEX_LEN = 64
|
||||
|
||||
|
||||
def _orphan_attachment_conditions() -> list:
|
||||
"""PostAttachment rows belonging to nothing: both FKs nulled by a deleted
|
||||
post AND a deleted artist. A row with post_id NULL but an artist_id is the
|
||||
deliberate filesystem-import case (importer._capture_attachment writes it
|
||||
that way) and is NOT an orphan — it is still attributed."""
|
||||
return [
|
||||
PostAttachment.post_id.is_(None),
|
||||
PostAttachment.artist_id.is_(None),
|
||||
]
|
||||
|
||||
|
||||
def _is_sha_named(name: str) -> bool:
|
||||
"""True when `name` starts with a 64-char lowercase-hex sha256."""
|
||||
if len(name) < _SHA256_HEX_LEN:
|
||||
return False
|
||||
head = name[:_SHA256_HEX_LEN]
|
||||
return all(c in "0123456789abcdef" for c in head)
|
||||
|
||||
|
||||
def reclaim_orphaned_attachments(
|
||||
session: Session, *, images_root: Path, dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Prune unattributed PostAttachment rows, then unlink store blobs that no
|
||||
surviving row references.
|
||||
|
||||
Returns (same discovery keys either way, so the UI renders one shape):
|
||||
{"rows": int, # orphan rows found / deleted
|
||||
"files": int, # unreferenced blobs found / unlinked
|
||||
"bytes": int, # their total size
|
||||
"scanned": int, # blobs examined
|
||||
"skipped_recent": int, # blobs under the min-age guard
|
||||
"files_failed": int, # unlink raised (apply only)
|
||||
"partial": bool} # walk hit the time budget
|
||||
|
||||
dry_run computes exactly what the apply would do and mutates nothing — the
|
||||
surviving-sha set is derived by NEGATING the same orphan predicate the
|
||||
delete uses, so the preview cannot disagree with the apply (rule 93).
|
||||
"""
|
||||
started = time.monotonic()
|
||||
orphan_conds = _orphan_attachment_conditions()
|
||||
|
||||
if dry_run:
|
||||
rows = session.execute(
|
||||
select(func.count(PostAttachment.id)).where(*orphan_conds)
|
||||
).scalar_one()
|
||||
else:
|
||||
rows = session.execute(
|
||||
delete(PostAttachment).where(*orphan_conds)
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
|
||||
# Shas that still have a home. In the apply path the orphan rows are already
|
||||
# gone, so `NOT orphan` is redundant but harmless; in the dry-run path it is
|
||||
# what makes the projection honest about blobs the delete would free. One
|
||||
# predicate, one query, both modes.
|
||||
surviving_shas = set(session.execute(
|
||||
select(PostAttachment.sha256).where(~and_(*orphan_conds)).distinct()
|
||||
).scalars())
|
||||
|
||||
root = Path(images_root) / "attachments"
|
||||
cutoff = (
|
||||
datetime.now(UTC).timestamp()
|
||||
- _ATTACHMENT_ORPHAN_MIN_AGE_HOURS * 3600
|
||||
)
|
||||
files = 0
|
||||
freed_bytes = 0
|
||||
scanned = 0
|
||||
skipped_recent = 0
|
||||
files_failed = 0
|
||||
partial = False
|
||||
|
||||
if root.is_dir():
|
||||
for path in root.rglob("*"):
|
||||
if time.monotonic() - started >= _ATTACHMENT_RECLAIM_BUDGET_SECONDS:
|
||||
partial = True
|
||||
break
|
||||
# .partial staging files belong to cleanup_orphaned_temp_files —
|
||||
# leave them alone rather than racing an in-flight store().
|
||||
if path.suffix in (".part", ".partial") or not path.is_file():
|
||||
continue
|
||||
if not _is_sha_named(path.name):
|
||||
continue
|
||||
scanned += 1
|
||||
sha = path.name[:_SHA256_HEX_LEN]
|
||||
if sha in surviving_shas:
|
||||
continue
|
||||
try:
|
||||
st = path.stat()
|
||||
if st.st_mtime >= cutoff:
|
||||
skipped_recent += 1
|
||||
continue
|
||||
size = st.st_size
|
||||
if not dry_run:
|
||||
path.unlink()
|
||||
files += 1
|
||||
freed_bytes += size
|
||||
except OSError as exc:
|
||||
files_failed += 1
|
||||
log.warning("reclaim_orphaned_attachments: %s: %s", path, exc)
|
||||
|
||||
if not dry_run and (rows or files):
|
||||
log.info(
|
||||
"attachment reclaim: %d orphan row(s) deleted, %d blob(s) unlinked "
|
||||
"(%d bytes), %d failed, partial=%s",
|
||||
rows, files, freed_bytes, files_failed, partial,
|
||||
)
|
||||
return {
|
||||
"rows": rows,
|
||||
"files": files,
|
||||
"bytes": freed_bytes,
|
||||
"scanned": scanned,
|
||||
"skipped_recent": skipped_recent,
|
||||
"files_failed": files_failed,
|
||||
"partial": partial,
|
||||
}
|
||||
|
||||
@@ -409,3 +409,31 @@ def rescan_series_suggestions_task(self, after_post_id: int = 0) -> dict:
|
||||
)
|
||||
rescan_series_suggestions_task.delay(summary["resume_after_id"])
|
||||
return summary
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.reclaim_orphaned_attachments_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
# The service stops walking at its own 900s budget and reports partial, so
|
||||
# these limits are the backstop for a wedged filesystem (NFS stall), not the
|
||||
# expected exit. Comfortably above the budget so a normal run always returns
|
||||
# its summary rather than being killed mid-walk.
|
||||
soft_time_limit=1200, time_limit=1500, # 20 min / 25 min
|
||||
)
|
||||
def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
|
||||
"""Reclaim unattributed PostAttachment rows and the store blobs nothing
|
||||
references any more (#3068). dry_run (the default) returns the projection
|
||||
without touching rows or files; apply deletes the orphan rows, then unlinks
|
||||
every blob no surviving row references.
|
||||
|
||||
Defaults to the SAFE preview — unlike the other tasks here, whose apply is
|
||||
reversible-ish or scoped; this one deletes files. Operator-triggered only,
|
||||
never on a beat: an unattended sweep that unlinks blobs is not something to
|
||||
run without someone reading the projection first."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return cleanup_service.reclaim_orphaned_attachments(
|
||||
session, images_root=IMAGES_ROOT, dry_run=dry_run,
|
||||
)
|
||||
|
||||
@@ -173,6 +173,12 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# task-name override beats the queue threshold whatever queue the row records
|
||||
# (it recorded 'default' before the celery_signals fix → download). 65 = 60+5.
|
||||
"backend.app.tasks.external.fetch_external_link": 65,
|
||||
# Attachment reclaim walks the whole sha-addressed store; the service caps
|
||||
# itself at a 900s budget and reports partial, but the task's hard limit is
|
||||
# 25 min for a wedged filesystem (NFS stall). Same phantom-flag class as the
|
||||
# external-fetch entry above — without an override a healthy in-flight walk
|
||||
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
<template>
|
||||
<!-- #3068: attachment reclamation. PostAttachment's FKs are both SET NULL, so
|
||||
a deleted post or artist leaves the row behind; and the store is
|
||||
sha-addressed, so one blob backs many rows and deleting a row never freed
|
||||
its file. Nothing swept either. Preview first, then apply (destructive:
|
||||
unlinks files). -->
|
||||
<MaintenanceTile
|
||||
icon="mdi-paperclip-off"
|
||||
title="Reclaim orphaned attachments"
|
||||
blurb="Remove attachment records belonging to nothing, and the files nothing references."
|
||||
destructive
|
||||
:open="applying || previewing"
|
||||
>
|
||||
<p class="text-body-2 mb-3">
|
||||
Attachment records survive the post and artist they belonged to, and the
|
||||
files behind them are shared between records — so a deleted record never
|
||||
freed its file on its own. This finds records attributed to
|
||||
<strong>neither</strong> a post nor an artist, and files in the attachment
|
||||
store that <strong>no remaining record</strong> points at.
|
||||
<strong>Preview</strong> first; <strong>Apply</strong> deletes those
|
||||
records and unlinks those files. Files written in the last few hours are
|
||||
always left alone, so an in-progress download is never caught mid-write.
|
||||
</p>
|
||||
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" variant="tonal" rounded="pill"
|
||||
:loading="previewing" :disabled="applying" @click="preview"
|
||||
>
|
||||
<v-icon start>mdi-magnify</v-icon> Preview
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="error" rounded="pill"
|
||||
:loading="applying"
|
||||
:disabled="previewing || !canApply"
|
||||
@click="confirmOpen = true"
|
||||
>
|
||||
<v-icon start>mdi-paperclip-off</v-icon> Apply
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||
density="comfortable"
|
||||
>
|
||||
<span v-if="applied">
|
||||
Deleted {{ summary.rows }} orphaned record(s) and unlinked
|
||||
{{ summary.files }} file(s), reclaiming {{ humanBytes(summary.bytes) }}.
|
||||
</span>
|
||||
<span v-else-if="hasWork">
|
||||
{{ summary.rows }} orphaned record(s) and {{ summary.files }}
|
||||
unreferenced file(s) — {{ humanBytes(summary.bytes) }} reclaimable.
|
||||
Click <strong>Apply</strong> to remove them.
|
||||
</span>
|
||||
<span v-else>Nothing to reclaim — every attachment is accounted for.</span>
|
||||
|
||||
<!-- Both of these change what the numbers MEAN, so they are stated
|
||||
whenever they are non-zero rather than hidden in a tooltip. -->
|
||||
<div v-if="summary.files_failed" class="mt-1 text-caption">
|
||||
{{ summary.files_failed }} file(s) could not be read or removed — see
|
||||
the worker log.
|
||||
</div>
|
||||
<div v-if="summary.partial" class="mt-1 text-caption">
|
||||
Stopped early at the time limit; some of the store was not examined.
|
||||
Run it again to continue.
|
||||
</div>
|
||||
</v-alert>
|
||||
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Maintenance" />
|
||||
|
||||
<v-dialog v-model="confirmOpen" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title>Reclaim orphaned attachments?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
This permanently deletes
|
||||
<strong>{{ summary?.rows ?? 0 }}</strong> attachment record(s) and
|
||||
unlinks <strong>{{ summary?.files ?? 0 }}</strong> file(s)
|
||||
({{ humanBytes(summary?.bytes) }}). Only files that no remaining
|
||||
record points at are removed, so nothing still attached to a post
|
||||
is affected.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||
<v-btn color="error" @click="apply">Reclaim</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
// Walks the whole attachment store, so it can run for minutes on a large
|
||||
// library — the service caps itself at 900s and reports `partial`. 150 polls
|
||||
// × 2s ≈ 5m of foreground waiting; past that the composable hands off to the
|
||||
// task dashboard rather than spinning forever.
|
||||
const { previewing, applying, summary, applied, preview, apply: applyTask } = useMaintenanceTask({
|
||||
endpoint: '/api/admin/maintenance/reclaim-attachments',
|
||||
storageKey: 'fc.maint.reclaimAttachments',
|
||||
appliedToast: 'Orphaned attachments reclaimed',
|
||||
maxPolls: 150,
|
||||
})
|
||||
|
||||
const hasWork = computed(
|
||||
() => !!summary.value && (summary.value.rows > 0 || summary.value.files > 0),
|
||||
)
|
||||
const canApply = computed(() => hasWork.value && !applied.value)
|
||||
const summaryType = computed(() => {
|
||||
if (applied.value) return 'success'
|
||||
return hasWork.value ? 'info' : 'success'
|
||||
})
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
applyTask()
|
||||
}
|
||||
</script>
|
||||
@@ -102,6 +102,7 @@
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
@@ -122,14 +123,6 @@ const summaryType = computed(() => {
|
||||
return summary.value && summary.value.matched > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
|
||||
@@ -78,6 +78,7 @@
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useMaintenanceTask } from '../../composables/useMaintenanceTask.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
@@ -98,14 +99,6 @@ const summaryType = computed(() => {
|
||||
return summary.value && summary.value.redundant > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// The confirm dialog gates the destructive apply; close it, then run.
|
||||
function apply () {
|
||||
confirmOpen.value = false
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Human-readable byte sizes for maintenance summaries ("2.4 GB reclaimable").
|
||||
//
|
||||
// Promoted out of the cleanup cards, which had grown byte-identical private
|
||||
// copies (VideoDedupCard, GatedPurgeCard) and were about to grow a third for
|
||||
// the attachment reclaim. Binary units (1 KB = 1024 B) — these numbers come
|
||||
// from st_size / SUM(size_bytes), so they describe disk, not marketing.
|
||||
//
|
||||
// NOT the same shape as the `formatBytes` helpers in SystemStatsCards,
|
||||
// BackupRunsTable and PostCard — those differ in units, precision and
|
||||
// zero-handling. Left alone deliberately rather than force-fitted here.
|
||||
export function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
@@ -19,14 +19,16 @@
|
||||
</section>
|
||||
|
||||
<section class="fc-section">
|
||||
<h3 class="fc-section__title">Duplicates & posts</h3>
|
||||
<h3 class="fc-section__title">Duplicates & leftovers</h3>
|
||||
<p class="fc-section__hint">
|
||||
Tidy post records, duplicates and locked-preview leftovers.
|
||||
Tidy post records, duplicates, locked-preview leftovers and attachments
|
||||
that outlived what they belonged to.
|
||||
</p>
|
||||
<div class="fc-tile-grid">
|
||||
<PostMaintenanceCard />
|
||||
<VideoDedupCard />
|
||||
<GatedPurgeCard />
|
||||
<AttachmentReclaimCard />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -60,6 +62,7 @@ import SingleColorAuditCard from '../components/cleanup/SingleColorAuditCard.vue
|
||||
import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
|
||||
import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
|
||||
import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
|
||||
import AttachmentReclaimCard from '../components/settings/AttachmentReclaimCard.vue'
|
||||
import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
|
||||
import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
|
||||
</script>
|
||||
|
||||
@@ -677,3 +677,34 @@ async def test_reset_content_tagging_apply_requires_confirm_token(client, db):
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert (await resp.get_json())["deleted"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_reclaim_attachments_defaults_to_preview(client, monkeypatch):
|
||||
"""Unlike the other maintenance triggers, this one's apply unlinks FILES —
|
||||
so an empty body must mean preview, not apply."""
|
||||
from backend.app.tasks import admin as admin_tasks
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
admin_tasks.reclaim_orphaned_attachments_task, "delay", _fake_delay(calls)
|
||||
)
|
||||
resp = await client.post("/api/admin/maintenance/reclaim-attachments", json={})
|
||||
assert resp.status_code == 202
|
||||
assert (await resp.get_json())["task_id"] == "task-xyz"
|
||||
assert calls[0][1] == {"dry_run": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_reclaim_attachments_threads_apply(client, monkeypatch):
|
||||
from backend.app.tasks import admin as admin_tasks
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
admin_tasks.reclaim_orphaned_attachments_task, "delay", _fake_delay(calls)
|
||||
)
|
||||
resp = await client.post(
|
||||
"/api/admin/maintenance/reclaim-attachments", json={"dry_run": False},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
assert calls[0][1] == {"dry_run": False}
|
||||
|
||||
@@ -5,6 +5,7 @@ side effects use tmp_path. Assertions on mutated rows use COLUMN
|
||||
SELECTS per reference_async_coredml_test_assertions — never
|
||||
re-read ORM attributes after a service mutates and re-fetches.
|
||||
"""
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
@@ -1081,3 +1082,176 @@ def test_reconcile_preserves_from_attachment_on_provenance_collision(db_sync, tm
|
||||
.where(ImageProvenance.image_record_id == img_id)
|
||||
).all()
|
||||
assert rows == [(native_id, att_id)]
|
||||
|
||||
|
||||
# --- reclaim_orphaned_attachments -----------------------------------
|
||||
|
||||
|
||||
def _store_blob(root, sha, *, ext=".pdf", age_hours=48, data=b"blob"):
|
||||
"""Write a file into the sha-addressed attachment store, aged past the
|
||||
min-age guard by default."""
|
||||
d = root / "attachments" / sha[:3]
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
p = d / f"{sha}{ext}"
|
||||
p.write_bytes(data)
|
||||
old = datetime.now(UTC).timestamp() - age_hours * 3600
|
||||
os.utime(p, (old, old))
|
||||
return p
|
||||
|
||||
|
||||
def _attachment(db_sync, *, sha, post=None, artist=None):
|
||||
att = PostAttachment(
|
||||
post_id=post.id if post else None,
|
||||
artist_id=artist.id if artist else None,
|
||||
sha256=sha, path=f"/store/{sha[:3]}/f.pdf",
|
||||
original_filename="f.pdf", ext=".pdf", size_bytes=4,
|
||||
)
|
||||
db_sync.add(att)
|
||||
db_sync.flush()
|
||||
return att
|
||||
|
||||
|
||||
def test_reclaim_attachments_dry_run_projects_without_mutating(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="recl-dry")
|
||||
p = Post(artist_id=a.id, external_post_id="rd-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
kept_sha, orphan_sha = "aa11".ljust(64, "0"), "bb22".ljust(64, "0")
|
||||
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
||||
_attachment(db_sync, sha=orphan_sha) # both FKs NULL → orphan
|
||||
db_sync.commit()
|
||||
kept_blob = _store_blob(tmp_path, kept_sha)
|
||||
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
assert result["rows"] == 1
|
||||
assert result["files"] == 1
|
||||
assert result["bytes"] == orphan_blob.stat().st_size
|
||||
|
||||
# Nothing actually happened.
|
||||
assert kept_blob.exists() and orphan_blob.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count(PostAttachment.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
|
||||
def test_reclaim_attachments_apply_deletes_rows_and_unlinks_blobs(db_sync, tmp_path):
|
||||
a = _make_artist(db_sync, slug="recl-apply")
|
||||
p = Post(artist_id=a.id, external_post_id="ra-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
kept_sha, orphan_sha = "cc33".ljust(64, "0"), "dd44".ljust(64, "0")
|
||||
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
||||
_attachment(db_sync, sha=orphan_sha)
|
||||
db_sync.commit()
|
||||
kept_blob = _store_blob(tmp_path, kept_sha)
|
||||
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 1
|
||||
assert result["files"] == 1
|
||||
|
||||
assert kept_blob.exists() # still referenced
|
||||
assert not orphan_blob.exists() # nothing points at it any more
|
||||
surviving = db_sync.execute(select(PostAttachment.sha256)).scalars().all()
|
||||
assert surviving == [kept_sha]
|
||||
|
||||
|
||||
def test_reclaim_attachments_preview_matches_apply(db_sync, tmp_path):
|
||||
"""Rule 93 — the dry-run's numbers are what the apply does. The projection
|
||||
has to negate the orphan predicate to be honest about blobs the delete is
|
||||
about to free, so this is the assertion that catches getting that backwards.
|
||||
"""
|
||||
orphan_sha = "ee55".ljust(64, "0")
|
||||
_attachment(db_sync, sha=orphan_sha)
|
||||
db_sync.commit()
|
||||
_store_blob(tmp_path, orphan_sha)
|
||||
|
||||
projected = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
applied = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
for key in ("rows", "files", "bytes"):
|
||||
assert projected[key] == applied[key], key
|
||||
assert applied["rows"] == 1 and applied["files"] == 1
|
||||
|
||||
|
||||
def test_reclaim_attachments_keeps_shared_blob_while_any_row_remains(db_sync, tmp_path):
|
||||
"""The refcount case this whole sweep exists for: one sha-addressed blob
|
||||
backs several rows, so deleting SOME of them must not free the file."""
|
||||
a = _make_artist(db_sync, slug="recl-shared")
|
||||
p = Post(artist_id=a.id, external_post_id="rs-1")
|
||||
db_sync.add(p)
|
||||
db_sync.flush()
|
||||
sha = "ff66".ljust(64, "0")
|
||||
_attachment(db_sync, sha=sha, post=p, artist=a) # attributed — survives
|
||||
_attachment(db_sync, sha=sha) # orphan — deleted
|
||||
db_sync.commit()
|
||||
blob = _store_blob(tmp_path, sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 1 # the orphan row went
|
||||
assert result["files"] == 0 # the blob did NOT
|
||||
assert blob.exists()
|
||||
|
||||
|
||||
def test_reclaim_attachments_spares_filesystem_import_rows(db_sync, tmp_path):
|
||||
"""post_id NULL with an artist_id is the deliberate filesystem-import shape
|
||||
(importer._capture_attachment), not an orphan — it is still attributed."""
|
||||
a = _make_artist(db_sync, slug="recl-fsimport")
|
||||
sha = "1177".ljust(64, "0")
|
||||
_attachment(db_sync, sha=sha, artist=a) # post NULL, artist set
|
||||
db_sync.commit()
|
||||
blob = _store_blob(tmp_path, sha)
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["rows"] == 0
|
||||
assert result["files"] == 0
|
||||
assert blob.exists()
|
||||
assert db_sync.execute(
|
||||
select(func.count(PostAttachment.id))
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
def test_reclaim_attachments_skips_recent_and_staging_files(db_sync, tmp_path):
|
||||
"""A blob is written BEFORE its row commits, so a just-stored file with no
|
||||
row is in-flight, not orphaned. `.partial` staging files belong to
|
||||
cleanup_orphaned_temp_files and must be left alone either way."""
|
||||
fresh_sha, staged_sha = "2288".ljust(64, "0"), "3399".ljust(64, "0")
|
||||
fresh = _store_blob(tmp_path, fresh_sha, age_hours=0)
|
||||
staged = _store_blob(tmp_path, staged_sha, ext=".pdf.partial")
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["files"] == 0
|
||||
assert result["skipped_recent"] == 1
|
||||
assert fresh.exists() and staged.exists()
|
||||
|
||||
|
||||
def test_reclaim_attachments_ignores_non_sha_named_files(db_sync, tmp_path):
|
||||
"""The walk must only judge files it can identify as store blobs — anything
|
||||
else under the root is none of its business."""
|
||||
d = tmp_path / "attachments" / "zzz"
|
||||
d.mkdir(parents=True)
|
||||
stray = d / "notes.txt"
|
||||
stray.write_text("not a blob")
|
||||
old = datetime.now(UTC).timestamp() - 48 * 3600
|
||||
os.utime(stray, (old, old))
|
||||
|
||||
result = cleanup_service.reclaim_orphaned_attachments(
|
||||
db_sync, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert result["files"] == 0
|
||||
assert stray.exists()
|
||||
|
||||
@@ -778,3 +778,17 @@ def test_vacuum_analyze_runs_over_high_churn_tables():
|
||||
|
||||
result = vacuum_analyze.apply().get()
|
||||
assert result["vacuumed"] == list(VACUUM_TABLES)
|
||||
|
||||
|
||||
def test_reclaim_attachments_stuck_threshold_exceeds_hard_time_limit():
|
||||
"""#883's invariant, applied to the attachment reclaim: a task whose stall
|
||||
threshold is under its own hard limit gets phantom-flagged 'RecoverySweep'
|
||||
while it is still healthily running."""
|
||||
from backend.app.tasks.admin import reclaim_orphaned_attachments_task
|
||||
from backend.app.tasks.maintenance import TASK_STUCK_THRESHOLD_MINUTES
|
||||
|
||||
hard_minutes = reclaim_orphaned_attachments_task.time_limit / 60
|
||||
override = TASK_STUCK_THRESHOLD_MINUTES[
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task"
|
||||
]
|
||||
assert override >= hard_minutes
|
||||
|
||||
Reference in New Issue
Block a user