feat: test a database backup without touching the install

"Test a backup" on the Backups card loads a dump into a separate, temporary
database on the same Postgres server, the same way a restore does, checks every
table against the row counts recorded with the dump, brings the copy up to this
version, and drops it. The live database is only read.

Every dump now records its row counts, read in the snapshot pg_dump dumps from
(pg_export_snapshot + --snapshot), so the comparison is exact. Off-site the
counts sit beside the dump as <name>.counts.json; locally they go in the
manifest file. A dump taken before this is compared with the live database
instead.

A test cut off by a restart is ended, and its database dropped, when the long
maintenance lane starts. A test doesn't block backups; a restore and a second
test wait for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

frontend/src/components/settings/BackupCard.vue

@@ -78,10 +78,15 @@
         @restore="onRestore"
         @delete="onDelete"
         @tag="onTag"
+        @test="(r) => testSection?.start({ runId: r.id })"
       />
 
       <v-divider class="my-4" />
 
+      <BackupTestSection ref="testSection" :bucket="offsite?.bucket || ''" @started="pollOnce" />
+
+      <v-divider class="my-4" />
+
       <!-- Off-site: images and attachments as one file per fingerprint -->
       <template v-if="offsite">
         <h3 class="fc-section-title">Images and attachments</h3>
@@ -159,43 +164,11 @@
         </v-alert>
         <BackupRunsTable :runs="store.restoreRuns" empty-text="No restores yet." />
 
-        <v-dialog v-model="dumpsOpen" max-width="560">
-          <v-card>
-            <v-card-title>Dumps in {{ offsite.bucket }}</v-card-title>
-            <v-card-text>
-              <v-alert
-                v-if="dumpsError" type="warning" variant="tonal" density="compact" class="mb-3"
-              >{{ dumpsError }}</v-alert>
-              <v-table v-else density="compact">
-                <thead>
-                  <tr><th>Taken</th><th class="text-right">Size</th><th /></tr>
-                </thead>
-                <tbody>
-                  <tr v-for="d in dumps" :key="d.key">
-                    <td :title="d.key">{{ dumpWhen(d.ts) }}</td>
-                    <td class="text-right" style="font-variant-numeric: tabular-nums;">
-                      {{ humanBytes(d.size_bytes) }}
-                    </td>
-                    <td class="text-right">
-                      <v-btn size="small" variant="text" color="warning" @click="onPickDump(d)">
-                        Restore…
-                      </v-btn>
-                    </td>
-                  </tr>
-                  <tr v-if="!dumps.length">
-                    <td colspan="3" class="text-center fc-muted py-4">
-                      The bucket holds no database dumps yet.
-                    </td>
-                  </tr>
-                </tbody>
-              </v-table>
-            </v-card-text>
-            <v-card-actions>
-              <v-spacer />
-              <v-btn variant="text" @click="dumpsOpen = false">Close</v-btn>
-            </v-card-actions>
-          </v-card>
-        </v-dialog>
+        <DumpPickerDialog
+          v-model="dumpsOpen" action="restore"
+          :bucket="offsite.bucket" :dumps="dumps" :error="dumpsError"
+          @pick="onPickDump"
+        />
       </template>
 
       <template v-else>
@@ -246,8 +219,10 @@ import { useSystemHealthStore } from '../../stores/systemHealth.js'
 import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
 import MaintenanceTile from '../common/MaintenanceTile.vue'
 import { humanBytes } from '../../utils/bytes.js'
-import { formatDateTime, formatRelative } from '../../utils/date.js'
+import { formatDumpTime, formatRelative } from '../../utils/date.js'
 import BackupRunsTable from './BackupRunsTable.vue'
+import BackupTestSection from './BackupTestSection.vue'
+import DumpPickerDialog from './DumpPickerDialog.vue'
 
 const store = useBackupStore()
 const healthStore = useSystemHealthStore()
@@ -313,6 +288,7 @@ const dumpsError = ref('')
 const dumps = ref([])
 const pickedDump = ref(null)
 const fetchTriggering = ref(false)
+const testSection = ref(null)
 // The notice that a restore was asked for stays up until its run is over.
 watch(runningRestore, (now, before) => {
   if (before && !now) restoreDispatched.value = false
@@ -324,14 +300,6 @@ function restoreLine(m) {
     + `${n('present')} already there, ${n('missing')} not in the bucket, ${n('failed')} failed`
 }
 
-// 20261107T134258Z → the reader's local date and time.
-function dumpWhen(ts) {
-  const iso = ts.replace(
-    /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/, '$1-$2-$3T$4:$5:$6Z',
-  )
-  return formatDateTime(iso)
-}
-
 async function openDumps() {
   dumpsLoading.value = true
   dumpsError.value = ''
@@ -354,7 +322,7 @@ function onPickDump(d) {
   confirmRunId.value = ''
   confirmToken.value = d.confirm
   confirmDescription.value = (
-    `Replaces this install's database with the dump taken ${dumpWhen(d.ts)}, `
+    `Replaces this install's database with the dump taken ${formatDumpTime(d.ts)}, `
     + 'then fetches the library\'s files from the bucket.'
   )
   confirmOpen.value = true
@@ -380,6 +348,7 @@ const COVERAGE_EVERY = 6
 function pollOnce() {
   if (document.hidden) return
   store.loadRuns('db')
+  store.loadRuns('test')
   if (offsite.value) {
     store.loadRuns('media')
     store.loadRuns('restore')

frontend/src/components/settings/BackupRunsTable.vue

@@ -47,7 +47,7 @@
           <span v-else class="fc-muted">—</span>
         </td>
         <td class="text-right">
-          <span v-if="r.offsite" class="fc-muted">—</span>
+          <span v-if="r.offsite || !actions" class="fc-muted">—</span>
           <KebabMenu v-else :label="`Actions for backup ${r.id}`">
             <v-list-item
               :title="r.tag ? 'Untag' : 'Tag…'"
@@ -57,6 +57,11 @@
               title="Restore…" :disabled="r.status !== 'ok'"
               @click="$emit('restore', r)"
             />
+            <v-list-item
+              v-if="r.kind === 'db'"
+              title="Test restore" :disabled="r.status !== 'ok'"
+              @click="$emit('test', r)"
+            />
             <v-list-item
               title="Delete…"
               @click="$emit('delete', r)"
@@ -96,8 +101,11 @@ import KebabMenu from '../common/KebabMenu.vue'
 defineProps({
   runs: { type: Array, default: () => [] },
   emptyText: { type: String, default: 'No backups yet.' },
+  // False for runs that are records, not backups (tests): no Tag, Restore
+  // or Delete menu.
+  actions: { type: Boolean, default: true },
 })
-defineEmits(['restore', 'delete', 'tag'])
+defineEmits(['restore', 'delete', 'tag', 'test'])
 
 // Which runs have their failed-file list open.
 const open = reactive({})

frontend/src/components/settings/BackupTestSection.vue

@@ -0,0 +1,197 @@
+<template>
+  <div>
+    <h3 class="fc-section-title">Test a backup</h3>
+    <p class="fc-muted text-body-2 mb-2">
+      Load a database dump into a separate, temporary database, check that every
+      table came back with the rows it held when the dump was taken, and bring it
+      up to this version. The temporary database is then deleted. This install's
+      database and library aren't touched.
+      <template v-if="store.databaseBytes">
+        It needs about {{ humanBytes(store.databaseBytes) }} free on the
+        database's disk while it runs.
+      </template>
+    </p>
+    <div class="fc-settings-row mb-3">
+      <v-btn
+        color="accent" variant="tonal" rounded="pill"
+        prepend-icon="mdi-database-check-outline"
+        :disabled="blocked || (!bucket && !newestLocal)" :loading="starting"
+        @click="startNewest"
+      >Test the newest dump</v-btn>
+      <v-btn
+        v-if="bucket"
+        variant="text" rounded="pill"
+        :disabled="blocked" :loading="dumpsLoading"
+        @click="openDumps"
+      >Choose a dump…</v-btn>
+    </div>
+
+    <v-alert
+      v-if="running || dispatched"
+      type="info" variant="tonal" density="compact" class="mb-3"
+    >
+      Testing the dump<template v-if="running?.manifest?.dump?.ts">
+        taken {{ formatDumpTime(running.manifest.dump.ts) }}</template>:
+      {{ phaseText }}.
+    </v-alert>
+
+    <!-- The newest finished test: its verdict, and every table on request. -->
+    <div v-if="last" class="mb-3">
+      <div class="d-flex align-start text-body-2">
+        <v-icon
+          size="small" class="mr-2 mt-1"
+          :color="last.status === 'ok' ? 'success' : 'error'"
+        >{{ last.status === 'ok' ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
+        <div>
+          <strong>Last test</strong>
+          ({{ formatRelative(last.finished_at) }}<template v-if="last.manifest?.dump?.ts">,
+            dump taken {{ formatDumpTime(last.manifest.dump.ts) }}</template>):
+          {{ last.manifest?.summary || last.error }}
+          <div v-if="upgradeLine" class="text-caption fc-muted">{{ upgradeLine }}</div>
+        </div>
+      </div>
+      <v-btn
+        v-if="tables.length"
+        size="x-small" variant="text" class="mt-1"
+        :append-icon="showTables ? 'mdi-chevron-up' : 'mdi-chevron-down'"
+        @click="showTables = !showTables"
+      >{{ showTables ? 'Hide' : 'Show' }} each table</v-btn>
+      <v-table v-if="showTables" density="compact" class="fc-test-tables mt-1">
+        <thead>
+          <tr>
+            <th>Table</th>
+            <th class="text-right">In the dump</th>
+            <th class="text-right">Restored</th>
+            <th class="text-right">Live now</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr
+            v-for="t in tables" :key="t.table"
+            :class="{ 'text-error': mismatched.has(t.table) }"
+          >
+            <td><code>{{ t.table }}</code></td>
+            <td class="text-right">{{ num(t.dump) }}</td>
+            <td class="text-right">{{ num(t.restored) }}</td>
+            <td class="text-right fc-muted">{{ num(t.live) }}</td>
+          </tr>
+        </tbody>
+      </v-table>
+    </div>
+
+    <BackupRunsTable :runs="store.testRuns" :actions="false" empty-text="No tests yet." />
+
+    <DumpPickerDialog
+      v-if="bucket"
+      v-model="dumpsOpen" action="test"
+      :bucket="bucket" :dumps="dumps" :error="dumpsError"
+      @pick="onPick"
+    />
+  </div>
+</template>
+
+<script setup>
+import { computed, ref, watch } from 'vue'
+
+import { useBackupStore } from '../../stores/backup.js'
+import { humanBytes } from '../../utils/bytes.js'
+import { formatDumpTime, formatRelative } from '../../utils/date.js'
+import BackupRunsTable from './BackupRunsTable.vue'
+import DumpPickerDialog from './DumpPickerDialog.vue'
+
+const props = defineProps({
+  // The off-site bucket's name when one is set up; otherwise the dumps on
+  // this server are tested.
+  bucket: { type: String, default: '' },
+})
+const emit = defineEmits(['started'])
+
+const store = useBackupStore()
+
+const running = computed(() => store.testRuns.find((r) => r.status === 'running'))
+const last = computed(() => store.testRuns.find((r) => r.status !== 'running'))
+const restoring = computed(() => store.restoreRuns.some((r) => r.status === 'restoring'))
+const blocked = computed(() => Boolean(running.value || restoring.value))
+const newestLocal = computed(() => store.dbRuns.find((r) => r.status === 'ok' && !r.offsite))
+
+const PHASES = {
+  fetching: 'downloading it from the bucket',
+  loading: 'loading it into a temporary database',
+  comparing: 'counting every table',
+  upgrading: 'bringing it up to this version',
+}
+const phaseText = computed(() => PHASES[running.value?.manifest?.phase] || 'starting')
+
+const tables = computed(() => last.value?.manifest?.tables || [])
+const mismatched = computed(() => new Set(last.value?.manifest?.mismatched || []))
+const showTables = ref(false)
+const upgradeLine = computed(() => {
+  const m = last.value?.manifest
+  if (!m?.alembic_now) return ''
+  return m.alembic_dump === m.alembic_now
+    ? 'Its schema is already this version\'s.'
+    : `Its schema was brought up from ${m.alembic_dump || 'none'} to ${m.alembic_now}.`
+})
+
+function num(n) {
+  return n == null ? '—' : n.toLocaleString()
+}
+
+// "Testing…" shows from the click until the run appears.
+const dispatched = ref(false)
+watch(running, (now) => { if (now) dispatched.value = false })
+
+const starting = ref(false)
+async function start(what) {
+  starting.value = true
+  try {
+    await store.testDump(what)
+    dispatched.value = true
+    emit('started')
+  } catch {
+    // store.lastError is shown on the card.
+  } finally {
+    starting.value = false
+  }
+}
+function startNewest() {
+  start(props.bucket ? {} : { runId: newestLocal.value.id })
+}
+
+const dumpsOpen = ref(false)
+const dumpsLoading = ref(false)
+const dumpsError = ref('')
+const dumps = ref([])
+async function openDumps() {
+  dumpsLoading.value = true
+  dumpsError.value = ''
+  try {
+    dumps.value = await store.listOffsiteDumps()
+  } catch (e) {
+    dumps.value = []
+    dumpsError.value = e.message
+  } finally {
+    dumpsLoading.value = false
+    dumpsOpen.value = true
+  }
+}
+function onPick(d) {
+  dumpsOpen.value = false
+  start({ key: d.key })
+}
+
+defineExpose({ start })
+</script>
+
+<style scoped>
+.fc-section-title {
+  font-family: 'Fraunces', Georgia, serif;
+  font-size: 16px; font-weight: 500;
+  color: rgb(var(--v-theme-on-surface));
+  margin-bottom: 8px;
+}
+.fc-settings-row {
+  display: flex; gap: 12px; align-items: center; flex-wrap: wrap;
+}
+.fc-test-tables { background: transparent; font-variant-numeric: tabular-nums; }
+</style>

frontend/src/components/settings/DumpPickerDialog.vue

@@ -0,0 +1,72 @@
+<template>
+  <!-- The database dumps in the off-site bucket, newest first, each with one
+       action: restore it, or test it. -->
+  <v-dialog :model-value="modelValue" max-width="560" @update:model-value="$emit('update:modelValue', $event)">
+    <v-card>
+      <v-card-title>Dumps in {{ bucket }}</v-card-title>
+      <v-card-text>
+        <v-alert
+          v-if="error" type="warning" variant="tonal" density="compact" class="mb-3"
+        >{{ error }}</v-alert>
+        <template v-else>
+          <v-table density="compact">
+            <thead>
+              <tr><th>Taken</th><th class="text-right">Size</th><th /></tr>
+            </thead>
+            <tbody>
+              <tr v-for="d in dumps" :key="d.key">
+                <td :title="d.key">
+                  {{ formatDumpTime(d.ts) }}
+                  <v-icon
+                    v-if="d.counted" size="x-small" class="ml-1" color="success"
+                    title="Its row counts were recorded, so a test checks every table exactly"
+                  >mdi-check-decagram-outline</v-icon>
+                </td>
+                <td class="text-right fc-tabular">{{ humanBytes(d.size_bytes) }}</td>
+                <td class="text-right">
+                  <v-btn
+                    size="small" variant="text" :color="action === 'restore' ? 'warning' : 'accent'"
+                    @click="$emit('pick', d)"
+                  >{{ action === 'restore' ? 'Restore…' : 'Test' }}</v-btn>
+                </td>
+              </tr>
+              <tr v-if="!dumps.length">
+                <td colspan="3" class="text-center fc-muted py-4">
+                  The bucket holds no database dumps yet.
+                </td>
+              </tr>
+            </tbody>
+          </v-table>
+          <p v-if="action === 'test' && dumps.some((d) => d.counted)" class="fc-muted text-caption mt-2">
+            <v-icon size="x-small" color="success">mdi-check-decagram-outline</v-icon>
+            marks a dump whose row counts were recorded as it was taken. A dump
+            without them is compared with the live database instead.
+          </p>
+        </template>
+      </v-card-text>
+      <v-card-actions>
+        <v-spacer />
+        <v-btn variant="text" @click="$emit('update:modelValue', false)">Close</v-btn>
+      </v-card-actions>
+    </v-card>
+  </v-dialog>
+</template>
+
+<script setup>
+import { humanBytes } from '../../utils/bytes.js'
+import { formatDumpTime } from '../../utils/date.js'
+
+defineProps({
+  modelValue: { type: Boolean, default: false },
+  bucket: { type: String, default: '' },
+  dumps: { type: Array, default: () => [] },
+  error: { type: String, default: '' },
+  // 'restore' or 'test': what choosing a dump does.
+  action: { type: String, default: 'restore' },
+})
+defineEmits(['update:modelValue', 'pick'])
+</script>
+
+<style scoped>
+.fc-tabular { font-variant-numeric: tabular-nums; }
+</style>

frontend/src/stores/backup.js

@@ -12,15 +12,17 @@ export const useBackupStore = defineStore('backup', () => {
   const restoreRuns = ref([])
   // The weekly check of the bucket against what was sent.
   const reconcileRuns = ref([])
+  // Test restores: a dump loaded into a temporary database and checked.
+  const testRuns = ref([])
   const settings = ref(null)
   const loading = ref({
     dbRuns: false, imagesRuns: false, mediaRuns: false, restoreRuns: false,
-    reconcileRuns: false, settings: false,
+    reconcileRuns: false, testRuns: false, settings: false,
   })
   const lastError = ref(null)
   const RUNS = {
     db: dbRuns, images: imagesRuns, media: mediaRuns, restore: restoreRuns,
-    reconcile: reconcileRuns,
+    reconcile: reconcileRuns, test: testRuns,
   }
 
   async function loadRuns(kind) {
@@ -149,12 +151,34 @@ export const useBackupStore = defineStore('backup', () => {
   }
 
   // Restoring from the bucket. Dumps arrive newest first, each with the
-  // `confirm` text a restore of it asks for.
+  // `confirm` text a restore of it asks for, and `counted` when its row
+  // counts were recorded beside it.
+  // How big the live database is: a test restore needs about that much free
+  // space on the database's disk while it runs.
+  const databaseBytes = ref(null)
+
   async function listOffsiteDumps() {
     const body = await api.get('/api/system/backup/offsite/dumps')
+    databaseBytes.value = body.database_bytes ?? null
     return body.dumps || []
   }
 
+  // Test a dump without touching this install: from the bucket by key (the
+  // newest when null), or one kept on this server by its run id.
+  async function testDump({ key = null, runId = null } = {}) {
+    lastError.value = null
+    try {
+      if (runId != null) {
+        await api.post(`/api/system/backup/runs/${runId}/test`, { body: {} })
+      } else {
+        await api.post('/api/system/backup/offsite/test', { body: key ? { key } : {} })
+      }
+    } catch (e) {
+      lastError.value = e.message
+      throw e
+    }
+  }
+
   async function restoreFromOffsite(key, confirm) {
     lastError.value = null
     try {
@@ -176,8 +200,9 @@ export const useBackupStore = defineStore('backup', () => {
   }
 
   return {
-    dbRuns, imagesRuns, mediaRuns, restoreRuns, reconcileRuns, mediaCoverage, loadMediaCoverage,
-    listOffsiteDumps, restoreFromOffsite, fetchMissingFiles,
+    dbRuns, imagesRuns, mediaRuns, restoreRuns, reconcileRuns, testRuns,
+    mediaCoverage, loadMediaCoverage,
+    listOffsiteDumps, databaseBytes, restoreFromOffsite, fetchMissingFiles, testDump,
     settings, loading, lastError,
     loadRuns, triggerBackup, restore, deleteRun, setTag,
     loadSettings, patchSettings,

frontend/src/utils/date.js

@@ -61,3 +61,13 @@ export function formatLocalDate (iso) {
     year: 'numeric', month: 'short', day: 'numeric',
   })
 }
+
+// A dump's compact UTC stamp (20261107T134258Z, as in its file name) as the
+// reader's local date and time.
+export function formatDumpTime (ts) {
+  if (typeof ts !== 'string') return ''
+  const iso = ts.replace(
+    /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/, '$1-$2-$3T$4:$5:$6Z',
+  )
+  return formatDateTime(iso)
+}

frontend/test/components/backupTestSection.spec.js

@@ -0,0 +1,59 @@
+// @vitest-environment happy-dom
+import { describe, it, expect } from 'vitest'
+
+import BackupTestSection from '../../src/components/settings/BackupTestSection.vue'
+import { useBackupStore } from '../../src/stores/backup.js'
+import { mountWithStore } from '../support/mountComponent.js'
+
+const FINISHED = {
+  id: 9, kind: 'test', status: 'error', started_at: new Date().toISOString(),
+  finished_at: new Date().toISOString(), duration_seconds: 300, size_bytes: 2048,
+  error: '1 table(s) came back different: post (7 in the dump, 6 restored).',
+  offsite: null,
+  manifest: {
+    phase: 'done',
+    dump: { ts: '20261107T030000Z' },
+    summary: '1 table(s) came back different: post (7 in the dump, 6 restored).',
+    counted: true,
+    mismatched: ['post'],
+    alembic_dump: '0120', alembic_now: '0121',
+    tables: [
+      { table: 'artist', dump: 10, restored: 10, live: 12 },
+      { table: 'post', dump: 7, restored: 6, live: 7 },
+    ],
+  },
+}
+
+function mount(runs) {
+  return mountWithStore(BackupTestSection, () => {
+    useBackupStore().testRuns = runs
+  }, { props: { bucket: 'curator' } })
+}
+
+describe('BackupTestSection', () => {
+  it('leads with the last test\'s verdict and how its schema was brought up', () => {
+    const t = mount([FINISHED]).text()
+    expect(t).toContain('Last test')
+    expect(t).toContain('post (7 in the dump, 6 restored)')
+    expect(t).toContain('brought up from 0120 to 0121')
+    expect(t).not.toContain('artist')
+  })
+
+  it('lists every table on request, with the dump, restored and live counts', async () => {
+    const w = mount([FINISHED])
+    const btn = w.findAll('v-btn').find((b) => b.text().includes('each table'))
+    await btn.trigger('click')
+    const rows = w.findAll('tr').filter((r) => r.text().includes('post'))
+    expect(rows.some((r) => r.classes().includes('text-error'))).toBe(true)
+    expect(w.text()).toContain('artist')
+  })
+
+  it('says what a running test is doing', () => {
+    const running = {
+      ...FINISHED, id: 10, status: 'running', finished_at: null, error: null,
+      manifest: { phase: 'loading', dump: { ts: '20261107T030000Z' } },
+    }
+    const t = mount([running, FINISHED]).text()
+    expect(t).toContain('loading it into a temporary database')
+  })
+})