Backups: a daily dedupe-friendly tar to a folder or S3, with restore (#5918-#5921)

#5918 "Backup file": an uncompressed tar of per-table JSON Lines (primary-key
order, sorted keys) plus every attachment file, with fixed tar headers, a
dated name, and an atomic write through a dot-named .part file. Unchanged
rows are the same bytes every day, so restic off-site stores only the edits.
No credential ever enters a backup.

#5919 "Backup destinations": a folder, or an S3-compatible bucket through a
stdlib SigV4 put/list client (checked against AWS's published signatures),
with an upload limit. Test destination only lists, so an append-only key works.

#5920 "Backup job": daily at a chosen hour, Back up now, a 1/2/4 h retry
ladder, a one-hour deadline, an advisory lock, the last 30 runs, and an
overdue verdict on Settings -> Backups.

#5921 "Restore": python -m inkwell.restore FILE into an empty instance, an
integration test that backs up and restores, and docs/backups.md with
Fabled Archives as the worked example.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

frontend/src/components/BackupPanel.vue

@@ -0,0 +1,234 @@
+<script setup lang="ts">
+import { computed, onBeforeUnmount, onMounted, ref, watch } from "vue";
+import { api } from "../api/client";
+import { errorMessage } from "../api/errors";
+import { formatExact, timeAgo } from "../notes/datetime";
+import { roughSize } from "../size";
+import { useUiStore } from "../stores/ui";
+
+// Settings → Backups, above the fields: whether backups are happening, what is wrong
+// when they aren't, the two buttons, and the recent runs (#5920). Verdict first.
+
+interface Run {
+  started_at: string;
+  finished_at?: string;
+  trigger: "schedule" | "manual";
+  ok: boolean;
+  name?: string;
+  size_bytes?: number;
+  where?: string;
+  files?: number;
+  missing_files?: number;
+  error?: string;
+}
+interface Status {
+  state: "off" | "incomplete" | "waiting" | "ok" | "overdue" | "failing";
+  enabled: boolean;
+  where: string;
+  problems: string[];
+  running_since: string | null;
+  last_ok: Run | null;
+  failing_since: string | null;
+  next_run_at: string | null;
+  runs: Run[];
+}
+
+// Bumped by the page after a save, so the verdict follows the fields.
+const props = defineProps<{ refreshKey: number }>();
+const ui = useUiStore();
+
+const status = ref<Status | null>(null);
+const loadError = ref("");
+async function load(): Promise<void> {
+  try {
+    status.value = await api.get<Status>("/api/backups");
+    loadError.value = "";
+  } catch (e) {
+    loadError.value = errorMessage(e, "Couldn't load the backup status.");
+  }
+}
+watch(() => props.refreshKey, () => void load());
+
+// While a backup runs the page checks every few seconds, and stops by itself: when
+// the run ends, when the page closes, or after the run's own one-hour deadline.
+const POLL_MS = 3000;
+const POLL_LIMIT_MS = 65 * 60 * 1000;
+let poll: ReturnType<typeof setTimeout> | null = null;
+let pollStarted = 0;
+function stopPolling(): void {
+  if (poll !== null) clearTimeout(poll);
+  poll = null;
+}
+function pollWhileRunning(): void {
+  stopPolling();
+  if (!status.value?.running_since || Date.now() - pollStarted > POLL_LIMIT_MS) return;
+  poll = setTimeout(async () => {
+    try {
+      await load();
+    } finally {
+      pollWhileRunning();
+    }
+  }, POLL_MS);
+}
+watch(
+  () => status.value?.running_since,
+  (since) => {
+    if (since && poll === null) {
+      pollStarted = Date.now();
+      pollWhileRunning();
+    }
+  },
+);
+onBeforeUnmount(stopPolling);
+
+const starting = ref(false);
+async function backUpNow(): Promise<void> {
+  starting.value = true;
+  try {
+    await api.post("/api/backups/run");
+    ui.showToast("Backup started.");
+    await load();
+    // It may already be finished; if not, the watch above starts polling.
+  } catch (e) {
+    ui.showToast(errorMessage(e, "Couldn't start a backup."));
+  } finally {
+    starting.value = false;
+  }
+}
+
+const testing = ref(false);
+const testResult = ref<{ ok: boolean; text: string } | null>(null);
+async function testDestination(): Promise<void> {
+  testing.value = true;
+  testResult.value = null;
+  try {
+    const res = await api.post<{ message: string }>("/api/backups/test");
+    testResult.value = { ok: true, text: res.message };
+  } catch (e) {
+    testResult.value = { ok: false, text: errorMessage(e, "The destination didn't answer.") };
+  } finally {
+    testing.value = false;
+  }
+}
+
+function size(bytes?: number): string {
+  if (bytes === undefined) return "—";
+  return bytes >= 1024 ** 3 ? roughSize(bytes, "GB") : roughSize(bytes, "MB");
+}
+
+/** The one line that answers "are my notes backed up?". */
+const verdict = computed(() => {
+  const s = status.value;
+  if (!s) return null;
+  const last = s.runs[0];
+  if (s.running_since) return { tone: "neutral", text: `Backing up now, started ${timeAgo(s.running_since)}.` };
+  switch (s.state) {
+    case "off":
+      return { tone: "neutral", text: "Daily backups are off. Switch them on below; the first one runs a few minutes later." };
+    case "incomplete":
+      return { tone: "warn", text: `Backups can't run yet. ${s.problems.join(" ")}` };
+    case "waiting":
+      return { tone: "neutral", text: `The first backup to ${s.where} runs within a few minutes.` };
+    case "failing":
+      return {
+        tone: "bad",
+        text: `Backups have been failing since ${formatExact(s.failing_since)}: ${last?.error ?? "unknown error"}`,
+      };
+    case "overdue":
+      return {
+        tone: "warn",
+        text: `Overdue: the last good backup was ${timeAgo(s.last_ok?.finished_at ?? null)}.`,
+      };
+    default:
+      return {
+        tone: "good",
+        text: `Last backup ${timeAgo(s.last_ok?.finished_at ?? null)}, ${size(s.last_ok?.size_bytes)}, to ${s.last_ok?.where}.`,
+      };
+  }
+});
+const TONES: Record<string, string> = {
+  good: "border-green-300 bg-green-50 text-green-900 dark:border-green-900 dark:bg-green-950/40 dark:text-green-200",
+  warn: "border-amber-300 bg-amber-50 text-amber-900 dark:border-amber-800 dark:bg-amber-950/40 dark:text-amber-200",
+  bad: "border-red-300 bg-red-50 text-red-900 dark:border-red-900 dark:bg-red-950/40 dark:text-red-200",
+  neutral: "border-neutral-200 bg-neutral-50 text-neutral-700 dark:border-neutral-800 dark:bg-neutral-900 dark:text-neutral-300",
+};
+
+// A run that left out files the database names: the backup is good but not whole.
+const missingNote = computed(() => {
+  const n = status.value?.last_ok?.missing_files ?? 0;
+  return n ? `${n} attachment file${n === 1 ? " was" : "s were"} missing from disk and couldn't be included.` : "";
+});
+
+const showAll = ref(false);
+const runs = computed(() => (showAll.value ? status.value?.runs ?? [] : (status.value?.runs ?? []).slice(0, 5)));
+
+onMounted(load);
+</script>
+
+<template>
+  <section class="flex flex-col gap-4">
+    <p v-if="loadError" class="form-error">{{ loadError }}</p>
+    <p v-else-if="!status" class="list-empty">Loading…</p>
+    <template v-else-if="verdict">
+      <p class="rounded-lg border px-3 py-2 text-sm" :class="TONES[verdict.tone]">
+        {{ verdict.text }}
+        <span v-if="status.next_run_at && status.state !== 'incomplete'" class="block text-xs opacity-80">
+          Next scheduled backup: {{ formatExact(status.next_run_at) }}.
+        </span>
+      </p>
+      <p v-if="missingNote" class="text-xs text-amber-700 dark:text-amber-400">{{ missingNote }}</p>
+
+      <div class="flex flex-wrap items-center gap-2">
+        <button
+          type="button"
+          class="btn-sm"
+          :disabled="testing || status.problems.length > 0"
+          @click="testDestination"
+        >
+          {{ testing ? "Checking…" : "Test destination" }}
+        </button>
+        <button
+          type="button"
+          class="btn-sm"
+          :disabled="starting || !!status.running_since || status.problems.length > 0"
+          @click="backUpNow"
+        >
+          Back up now
+        </button>
+        <span
+          v-if="testResult"
+          class="text-xs"
+          :class="testResult.ok ? 'text-green-700 dark:text-green-400' : 'text-red-600 dark:text-red-400'"
+          >{{ testResult.text }}</span
+        >
+      </div>
+
+      <div v-if="status.runs.length" class="flex flex-col gap-1">
+        <h3 class="section-label">Recent backups</h3>
+        <ul class="flex flex-col divide-y divide-neutral-100 text-sm dark:divide-neutral-800">
+          <li v-for="r in runs" :key="r.started_at" class="flex flex-wrap items-baseline gap-x-3 gap-y-0.5 py-1.5">
+            <span
+              class="h-2 w-2 shrink-0 self-center rounded-full"
+              :class="r.ok ? 'bg-green-500' : 'bg-red-500'"
+              :aria-label="r.ok ? 'succeeded' : 'failed'"
+            ></span>
+            <span class="tabular-nums text-neutral-600 dark:text-neutral-300" :title="r.started_at">{{
+              formatExact(r.started_at)
+            }}</span>
+            <span class="text-xs text-neutral-400">{{ r.trigger === "manual" ? "by hand" : "scheduled" }}</span>
+            <span v-if="r.ok" class="ml-auto tabular-nums text-neutral-500">{{ size(r.size_bytes) }}</span>
+            <span v-if="!r.ok" class="w-full text-xs text-red-600 dark:text-red-400">{{ r.error }}</span>
+          </li>
+        </ul>
+        <button
+          v-if="status.runs.length > 5"
+          type="button"
+          class="self-start text-xs text-neutral-500 hover:text-neutral-800 dark:hover:text-neutral-200"
+          @click="showAll = !showAll"
+        >
+          {{ showAll ? "Show fewer" : `Show all ${status.runs.length}` }}
+        </button>
+      </div>
+    </template>
+  </section>
+</template>

frontend/src/notes/datetime.ts

@@ -1,3 +1,5 @@
+import { counted } from "../counted";
+
 // Reminder datetime helpers. Reminders are stored as ISO UTC; the editor's
 // <input type="datetime-local"> works in local time.
 
@@ -82,3 +84,29 @@ export function formatTrashCountdown(daysLeft: number | null): string {
   if (daysLeft === 1) return "1 day left";
   return `${daysLeft} days left`;
 }
+
+// How long ago a moment was, roughly: "4 minutes ago", "3 hours ago", "2 days ago".
+// For a status line, where the size of the gap matters more than the exact time.
+export function timeAgo(iso: string | null, now: number = Date.now()): string {
+  if (!iso) return "";
+  const minutes = Math.max(0, Math.round((now - new Date(iso).getTime()) / 60000));
+  if (minutes < 1) return "just now";
+  if (minutes < 60) return `${counted(minutes, "minute", "minutes")} ago`;
+  const hours = Math.round(minutes / 60);
+  if (hours < 48) return `${counted(hours, "hour", "hours")} ago`;
+  return `${counted(Math.round(hours / 24), "day", "days")} ago`;
+}
+
+// A moment exactly, in the reader's zone on a 24-hour clock, with its date so a row
+// still reads right after the day turns over: "Fri 17 Oct 04:00".
+export function formatExact(iso: string | null): string {
+  if (!iso) return "";
+  return new Date(iso).toLocaleString(undefined, {
+    weekday: "short",
+    day: "numeric",
+    month: "short",
+    hour: "2-digit",
+    minute: "2-digit",
+    hourCycle: "h23",
+  });
+}

frontend/src/views/SettingsView.vue

@@ -11,6 +11,7 @@ import InviteList from "../components/InviteList.vue";
 import AccountList from "../components/AccountList.vue";
 import ActivityList from "../components/ActivityList.vue";
 import GroupList from "../components/GroupList.vue";
+import BackupPanel from "../components/BackupPanel.vue";
 import { errorMessage } from "../api/errors";
 import { useUiStore } from "../stores/ui";
 
@@ -35,6 +36,7 @@ const TABS: Tab[] = [
   { id: "people", label: "People", groups: ["Access"] },
   { id: "email", label: "Email", groups: ["Email"] },
   { id: "security", label: "Security", groups: ["Security"] },
+  { id: "backups", label: "Backups", groups: ["Backups"] },
   { id: "activity", label: "Activity", groups: [] },
 ];
 const DEFAULT_TAB = "general";
@@ -61,6 +63,8 @@ const loadError = ref("");
 const saving = ref(false);
 const error = ref("");
 const saved = ref(false);
+// Bumped on every save, so the backup status above its fields follows them.
+const savedCount = ref(0);
 
 const isDirty = (it: ServerSetting) => it.value !== original.value[it.key];
 const tabItems = computed(() => items.value.filter((it) => tabOf(it.group) === tab.value));
@@ -77,10 +81,20 @@ watch(tab, () => {
   error.value = "";
 });
 
+/** A field that means nothing for the choice beside it: the bucket's fields while
+ *  backups go to a folder, and the folder while they go to a bucket. */
+function irrelevant(it: ServerSetting): boolean {
+  const dest = items.value.find((i) => i.key === "backup_destination")?.value;
+  if (it.key === "backup_folder") return dest === "s3";
+  if (it.key.startsWith("backup_s3_") || it.key === "backup_upload_limit_kbps") return dest !== "s3";
+  return false;
+}
+
 // The current tab's fields, under their group's heading, in registry order.
 const groups = computed(() => {
   const map = new Map<string, ServerSetting[]>();
   for (const it of tabItems.value) {
+    if (irrelevant(it)) continue;
     const arr = map.get(it.group) ?? [];
     arr.push(it);
     map.set(it.group, arr);
@@ -119,6 +133,7 @@ async function save() {
     snapshot(fresh);
     items.value = fresh.map((it) => (pending.has(it.key) ? { ...it, value: pending.get(it.key) as SettingValue } : it));
     saved.value = true;
+    savedCount.value++;
     await config.reload(); // header/site name reflects changes immediately
   } catch (e) {
     error.value = errorMessage(e, "Could not save settings.");
@@ -226,6 +241,9 @@ onMounted(load);
           <template v-else>Email is on, sending as {{ mailSender }}.</template>
         </p>
 
+        <!-- Backups lead with whether they are happening, as Email does. -->
+        <BackupPanel v-if="tab === 'backups'" :refresh-key="savedCount" />
+
         <form v-if="groups.length" class="flex flex-col gap-8" @submit.prevent="save">
           <section v-for="group in groups" :key="group.name" class="flex flex-col gap-5">
             <h2 class="section-label">{{ group.name }}</h2>