feat: images and attachments go to the off-site bucket once each, by fingerprint
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 25s
CI and images / extension-test (push) Successful in 18s
CI and images / backend-lint-and-test (push) Failing after 44s
CI and images / integration (push) Failing after 2m28s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 25s
CI and images / extension-test (push) Successful in 18s
CI and images / backend-lint-and-test (push) Failing after 44s
CI and images / integration (push) Failing after 2m28s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / build-agent (push) Skipped
With an off-site destination set up, the library's files are sent to the bucket's media/ folder as media/ab/cd/<sha256><ext>, beside the database dumps under database/. Each file goes once: a new backup_blob table records what each destination holds (migration 0121), and a run's work list is every image and attachment sha with no row yet, so runs are incremental and a new bucket starts its copy from the beginning. - The bucket checks each file against the sha recorded at import; a file over 1 GiB goes in parts and is checked before the upload is completed. A file missing or changed on disk is reported, not stored. - A run is chunked: each 45-minute chunk commits after every file and queues the next, so other long-maintenance jobs get turns. One run at a time. The stall sweep judges a media run by its progress heartbeat (backup_run.last_progress_at), not its start. - A file that can't be sent is listed on the run (status ok, with the error line) and tried again by the next run. A refused put for an object already in an append-only bucket counts as sent. - Nightly, with a destination set, runs the dump and then the media. - Weekly reconcile lists media/ and re-sends what the bucket lost, and adopts what it already holds (a restored database sends nothing twice). - Uploads use the bandwidth limit; read transactions end before transfers. - Backups card: with a destination, an "Images and attachments" section shows coverage (files and bytes in the bucket), "Send new files now", and the media runs. Without one, the local tarball section is unchanged. - Media run history keeps the latest 60 rows. Task #6879, milestone 586. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,8 +7,8 @@
|
||||
<p v-if="offsite" class="fc-muted text-body-2 mb-4">
|
||||
Database dumps stream to the <strong>{{ offsite.bucket }}</strong> bucket
|
||||
at {{ offsite.endpoint }}, with no copy kept on this server; the bucket's
|
||||
own retention thins old dumps. The images backup below still writes a
|
||||
tar+zstd to <code>/images/_backups/</code>, by hand only.
|
||||
own retention thins old dumps. Images and attachments go to the same
|
||||
bucket's <code>media/</code> folder, below.
|
||||
</p>
|
||||
<p v-else class="fc-muted text-body-2 mb-4">
|
||||
pg_dump for the database (fast, nightly-schedulable);
|
||||
@@ -48,7 +48,8 @@
|
||||
/>
|
||||
</div>
|
||||
<p v-if="offsite" class="fc-muted text-caption mb-3">
|
||||
Keep last applies to this list and to any dumps still on this server.
|
||||
Nightly sends the database, then any new images and attachments. Keep
|
||||
last applies to this list and to any dumps still on this server.
|
||||
</p>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
@@ -66,6 +67,38 @@
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<!-- Off-site: images and attachments as one file per fingerprint -->
|
||||
<template v-if="offsite">
|
||||
<h3 class="fc-section-title">Images and attachments</h3>
|
||||
<p class="fc-muted text-body-2 mb-2">
|
||||
Each file goes to the bucket's <code>media/</code> folder once, named by
|
||||
its fingerprint, and the bucket checks it against the fingerprint taken
|
||||
at import. Only files not sent yet go in each run. With Nightly on, they
|
||||
follow the database dump every night.
|
||||
</p>
|
||||
<div v-if="coverage" class="mb-3">
|
||||
<div class="text-body-2 mb-1" style="font-variant-numeric: tabular-nums;">
|
||||
{{ coverage.files_sent.toLocaleString() }} of
|
||||
{{ coverage.files_total.toLocaleString() }} files in the bucket
|
||||
({{ humanBytes(coverage.bytes_sent) }} of {{ humanBytes(coverage.bytes_total) }})
|
||||
</div>
|
||||
<v-progress-linear
|
||||
:model-value="coveragePct" color="accent" height="6" rounded
|
||||
/>
|
||||
</div>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-cloud-upload-outline" class="mb-3"
|
||||
:loading="mediaTriggering" :disabled="mediaRunning"
|
||||
@click="onTrigger('media')"
|
||||
>Send new files now</v-btn>
|
||||
<span v-if="mediaRunning" class="text-caption fc-muted ml-2">
|
||||
A run is going: {{ runningSent.toLocaleString() }} files sent so far.
|
||||
</span>
|
||||
<BackupRunsTable :runs="store.mediaRuns" />
|
||||
</template>
|
||||
|
||||
<template v-else>
|
||||
<!-- Images section -->
|
||||
<h3 class="fc-section-title">Images</h3>
|
||||
<div class="fc-settings-row mb-3">
|
||||
@@ -92,6 +125,7 @@
|
||||
@delete="onDelete"
|
||||
@tag="onTag"
|
||||
/>
|
||||
</template>
|
||||
<BackupConfirmModal
|
||||
v-model="confirmOpen"
|
||||
:action="confirmAction"
|
||||
@@ -109,6 +143,7 @@ import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
import { useBackupStore } from '../../stores/backup.js'
|
||||
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import BackupRunsTable from './BackupRunsTable.vue'
|
||||
|
||||
const store = useBackupStore()
|
||||
@@ -125,6 +160,16 @@ const offsite = computed(() => (
|
||||
store.destination?.configured ? store.destination : null
|
||||
))
|
||||
|
||||
const coverage = computed(() => store.mediaCoverage)
|
||||
const coveragePct = computed(() => {
|
||||
const c = coverage.value
|
||||
return c && c.files_total ? (100 * c.files_sent) / c.files_total : 0
|
||||
})
|
||||
const runningMedia = computed(() => store.mediaRuns.find((r) => r.status === 'running'))
|
||||
const mediaRunning = computed(() => Boolean(runningMedia.value))
|
||||
const runningSent = computed(() => runningMedia.value?.manifest?.sent || 0)
|
||||
|
||||
const mediaTriggering = ref(false)
|
||||
const dbTriggering = ref(false)
|
||||
const imagesTriggering = ref(false)
|
||||
const confirmOpen = ref(false)
|
||||
@@ -134,10 +179,19 @@ const confirmRunId = ref(0)
|
||||
const confirmDescription = ref('')
|
||||
|
||||
let pollId = null
|
||||
let polls = 0
|
||||
// Coverage counts every library file, so it refreshes every sixth poll
|
||||
// (30 s) rather than with the run lists.
|
||||
const COVERAGE_EVERY = 6
|
||||
function pollOnce() {
|
||||
if (document.hidden) return
|
||||
store.loadRuns('db')
|
||||
store.loadRuns('images')
|
||||
if (offsite.value) {
|
||||
store.loadRuns('media')
|
||||
if (polls++ % COVERAGE_EVERY === 0) store.loadMediaCoverage()
|
||||
} else {
|
||||
store.loadRuns('images')
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
@@ -151,7 +205,7 @@ onUnmounted(() => {
|
||||
})
|
||||
|
||||
async function onTrigger(kind) {
|
||||
const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
|
||||
const triggeringRef = { db: dbTriggering, images: imagesTriggering, media: mediaTriggering }[kind]
|
||||
triggeringRef.value = true
|
||||
try {
|
||||
await store.triggerBackup(kind)
|
||||
|
||||
Reference in New Issue
Block a user