Restore rolls back to a backup; a session can only file issues in another project #220

Merged
bvandeusen merged 15 commits from dev into main 2026-11-05 09:29:55 -05:00
12 changed files with 657 additions and 80 deletions
Showing only changes of commit 1f5e963c35 - Show all commits
+70 -1
View File
@@ -1,4 +1,4 @@
import { apiGet, apiPost, apiPut } from "@/api/client";
import { apiGet, apiPost, apiPostRaw, apiPut } from "@/api/client";
/**
* Scheduled backups — `/api/admin/backups`. One dated,
@@ -71,3 +71,72 @@ export function checkBackupDestination(
export function runBackupNow(): Promise<{ status: string }> {
return apiPost<{ status: string }>("/api/admin/backups/run", {});
}
// ── Restore: put the whole instance back to a full backup ─────────────────
export interface StoredBackup {
name: string;
size_bytes: number;
}
export interface RestoreCount {
kind: string;
now: number;
after: number;
}
export interface RestorePreview {
/** "fresh": an empty instance takes the backup. "rollback": this instance
* goes back to its own earlier backup. */
mode: "fresh" | "rollback";
exported_at: string | null;
counts: RestoreCount[];
}
export interface RestoreResult extends RestorePreview {
status: "ok";
restored: Record<string, number>;
safety_backup: { file: string; size_bytes: number; where: string; note?: string };
}
/** A backup at the destination, by name, or a file picked from this computer. */
export type RestoreFrom = { source: string } | { file: File };
// Reading, checking and writing the whole store is slow by design; bounded,
// but in minutes.
const RESTORE_TIMEOUT_MS = 10 * 60 * 1000;
export function listRestoreSources(): Promise<{ backups: StoredBackup[]; error?: string }> {
return apiGet("/api/admin/restore/sources", { timeoutMs: 60_000 });
}
async function restoreCall<T>(path: string, from: RestoreFrom, query: Record<string, string> = {}): Promise<T> {
const params = new URLSearchParams(query);
if ("source" in from) {
params.set("source", from.source);
return apiPostRaw<T>(`${path}?${params}`, null, null, { timeoutMs: RESTORE_TIMEOUT_MS });
}
const qs = params.toString() ? `?${params}` : "";
const isTar = from.file.name.toLowerCase().endsWith(".tar");
// The JSON download is parsed first, so a damaged file fails here, before
// the upload.
const body = isTar ? from.file : JSON.stringify(JSON.parse(await from.file.text()));
return apiPostRaw<T>(`${path}${qs}`, body, isTar ? "application/x-tar" : "application/json",
{ timeoutMs: RESTORE_TIMEOUT_MS });
}
export function previewRestore(from: RestoreFrom): Promise<RestorePreview> {
return restoreCall<RestorePreview>("/api/admin/restore/preview", from);
}
export function restoreBackup(from: RestoreFrom): Promise<RestoreResult> {
return restoreCall<RestoreResult>("/api/admin/restore", from, { confirm: "rollback" });
}
/** When a backup named `grimoire-20261015T020000Z.tar` was taken. */
export function backupTakenAt(name: string): Date | null {
const m = /(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/.exec(name);
if (!m) return null;
return new Date(Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4], +m[5], +m[6]));
}
+9
View File
@@ -156,6 +156,15 @@ export function apiPost<T>(path: string, body: unknown, opts?: RequestOpts): Pro
return request<T>(path, { method: "POST", headers: JSON_HEADERS, body: JSON.stringify(body) }, opts);
}
/** A POST whose body is sent as it is (a file, or text already serialised),
* under the same deadline and error handling as every other call. */
export function apiPostRaw<T>(
path: string, body: BodyInit | null, contentType: string | null, opts?: RequestOpts,
): Promise<T> {
const headers = contentType ? { "Content-Type": contentType } : undefined;
return request<T>(path, { method: "POST", headers, body }, opts);
}
export function apiPut<T>(path: string, body: unknown, opts?: RequestOpts): Promise<T> {
return request<T>(path, { method: "PUT", headers: JSON_HEADERS, body: JSON.stringify(body) }, opts);
}
@@ -0,0 +1,278 @@
<script setup lang="ts">
/**
* Restore: put the whole instance back to a full backup.
*
* Choose a backup (one at the backup destination, or a file from this
* computer), see what it would change, type the confirmation, run it. On a
* live instance this is a rollback to the instance's own earlier backup; on a
* new, empty one it takes the backup in. Either way the server backs up the
* current state first and signs everyone out afterwards.
*/
import { computed, onMounted, ref } from "vue";
import { apiErrorMessage } from "@/api/client";
import {
backupTakenAt, listRestoreSources, previewRestore, restoreBackup,
type RestoreFrom, type RestorePreview, type RestoreResult, type StoredBackup,
} from "@/api/backups";
import { formatBytes } from "@/utils/bytes";
const emit = defineEmits<{ close: [] }>();
const CONFIRM_WORDS = "roll back";
type Step = "choose" | "checking" | "preview" | "running" | "done";
const step = ref<Step>("choose");
const sources = ref<StoredBackup[]>([]);
const sourcesError = ref("");
const sourcesLoading = ref(true);
const chosen = ref<RestoreFrom | null>(null);
const chosenLabel = ref("");
const preview = ref<RestorePreview | null>(null);
const result = ref<RestoreResult | null>(null);
const error = ref("");
const typed = ref("");
const fileInput = ref<HTMLInputElement | null>(null);
const confirmed = computed(() => typed.value.trim().toLowerCase() === CONFIRM_WORDS);
function when(value: Date | string | null | undefined): string {
if (!value) return "an unknown date";
const date = typeof value === "string" ? new Date(value) : value;
return Number.isNaN(date.getTime()) ? "an unknown date" : date.toLocaleString();
}
onMounted(async () => {
try {
const answer = await listRestoreSources();
sources.value = answer.backups;
sourcesError.value = answer.error ?? "";
} catch (e) {
sourcesError.value = apiErrorMessage(e, "Could not list the backups at the destination.");
} finally {
sourcesLoading.value = false;
}
});
async function check(from: RestoreFrom, label: string) {
chosen.value = from;
chosenLabel.value = label;
error.value = "";
typed.value = "";
step.value = "checking";
try {
preview.value = await previewRestore(from);
step.value = "preview";
} catch (e) {
error.value = apiErrorMessage(e, "That backup could not be read.");
step.value = "choose";
}
}
function pickFile(event: Event) {
const file = (event.target as HTMLInputElement).files?.[0];
if (fileInput.value) fileInput.value.value = "";
if (file) check({ file }, file.name);
}
async function run() {
if (!chosen.value || !confirmed.value) return;
error.value = "";
step.value = "running";
try {
result.value = await restoreBackup(chosen.value);
step.value = "done";
} catch (e) {
error.value = apiErrorMessage(e, "The restore failed. Nothing was changed.");
step.value = "preview";
}
}
function finish() {
// Every session ended with the restore, this one included.
window.location.assign("/login");
}
</script>
<template>
<teleport to="body">
<div class="modal-overlay" @click="step === 'running' || step === 'done' ? null : emit('close')">
<div class="modal-card restore-card" @click.stop>
<template v-if="step === 'choose' || step === 'checking'">
<h3 class="modal-title">Restore from a backup</h3>
<p class="modal-message">
Puts the whole instance back to the moment the backup was taken, for every user.
You'll see what changes before anything happens.
</p>
<p v-if="error" class="restore-error" role="alert">{{ error }}</p>
<h4 class="restore-subhead">At the backup destination</h4>
<p v-if="sourcesLoading" class="restore-muted">Looking for backups…</p>
<p v-else-if="sourcesError" class="restore-muted">{{ sourcesError }}</p>
<p v-else-if="!sources.length" class="restore-muted">No backups there yet.</p>
<ul v-else class="restore-sources">
<li v-for="b in sources" :key="b.name">
<button
class="restore-source"
:disabled="step === 'checking'"
@click="check({ source: b.name }, when(backupTakenAt(b.name)))"
>
<span>{{ when(backupTakenAt(b.name)) }}</span>
<span class="restore-muted">{{ formatBytes(b.size_bytes) }}</span>
</button>
</li>
</ul>
<h4 class="restore-subhead">From this computer</h4>
<p class="restore-muted">
A backup file (.tar) or the Full Backup download (.json). Uploads are size-limited;
a large file restores from the destination above, or with
<code>python -m grimoire.restore</code>.
</p>
<div class="modal-actions restore-actions-left">
<button class="modal-btn" :disabled="step === 'checking'" @click="fileInput?.click()">
Choose a file…
</button>
<input ref="fileInput" type="file" accept=".json,.tar" class="hidden-file-input" @change="pickFile" />
</div>
<p v-if="step === 'checking'" class="restore-muted">Reading {{ chosenLabel }}…</p>
<div class="modal-actions">
<button class="modal-btn" @click="emit('close')">Cancel</button>
</div>
</template>
<template v-else-if="(step === 'preview' || step === 'running') && preview">
<h3 class="modal-title">
{{ preview.mode === "rollback" ? "Roll back to this backup?" : "Restore this backup?" }}
</h3>
<p class="modal-message">
<template v-if="preview.mode === 'rollback'">
Everything goes back to {{ when(preview.exported_at) }}, for every user. Anything made
since is removed, edits since are undone, and records deleted since come back.
</template>
<template v-else>
This instance has no records yet, so it takes the backup from {{ when(preview.exported_at) }} as it is.
</template>
The current state is backed up first, and everyone is signed out afterwards.
</p>
<table class="restore-counts">
<thead><tr><th></th><th>Now</th><th>After</th></tr></thead>
<tbody>
<tr v-for="c in preview.counts" :key="c.kind">
<td>{{ c.kind }}</td><td>{{ c.now }}</td><td>{{ c.after }}</td>
</tr>
</tbody>
</table>
<p v-if="error" class="restore-error" role="alert">{{ error }}</p>
<label class="restore-confirm">
Type <strong>{{ CONFIRM_WORDS }}</strong> to continue
<input v-model="typed" class="fs-input input" :disabled="step === 'running'" autocomplete="off" />
</label>
<div class="modal-actions">
<button class="modal-btn" :disabled="step === 'running'" @click="step = 'choose'">Back</button>
<button
class="modal-btn modal-btn-danger"
:disabled="!confirmed || step === 'running'"
@click="run"
>{{ step === "running" ? "Restoring…" : preview.mode === "rollback" ? "Roll back" : "Restore" }}</button>
</div>
</template>
<template v-else-if="step === 'done' && result">
<h3 class="modal-title">Restored</h3>
<p class="modal-message">
The instance is back to {{ when(result.exported_at) }}.
The state it replaced was saved as <code>{{ result.safety_backup.file }}</code>
in {{ result.safety_backup.where }}.
<template v-if="result.safety_backup.note"> {{ result.safety_backup.note }}</template>
Search is being rebuilt in the background. Everyone has been signed out, so sign in again.
</p>
<div class="modal-actions">
<button class="modal-btn modal-btn-primary" @click="finish">Sign in again</button>
</div>
</template>
</div>
</div>
</teleport>
</template>
<style scoped>
.restore-card {
max-width: 34rem;
max-height: 85vh;
overflow-y: auto;
}
.restore-subhead {
margin: 1rem 0 0.4rem;
font-size: 0.9rem;
}
.restore-muted {
font-size: 0.85rem;
color: var(--fs-text-secondary);
margin: 0 0 0.5rem;
}
.restore-error {
font-size: 0.85rem;
color: var(--fs-action-destructive);
margin: 0 0 0.75rem;
}
.restore-sources {
list-style: none;
margin: 0 0 0.5rem;
padding: 0;
max-height: 14rem;
overflow-y: auto;
border: 1px solid var(--fs-border-color);
border-radius: var(--fs-radius-sm);
}
.restore-source {
display: flex;
justify-content: space-between;
width: 100%;
padding: 0.45rem 0.7rem;
border: none;
border-bottom: 1px solid var(--fs-border-color);
background: none;
color: var(--fs-text-primary);
font: inherit;
font-size: 0.875rem;
text-align: left;
cursor: pointer;
}
.restore-sources li:last-child .restore-source {
border-bottom: none;
}
.restore-source:hover:not(:disabled) {
background: var(--fs-surface-hover);
}
.restore-actions-left {
justify-content: flex-start;
margin-bottom: 1rem;
}
.restore-counts {
width: 100%;
border-collapse: collapse;
font-size: 0.875rem;
margin: 0 0 1rem;
}
.restore-counts th,
.restore-counts td {
text-align: right;
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--fs-border-color);
}
.restore-counts th:first-child,
.restore-counts td:first-child {
text-align: left;
}
.hidden-file-input {
display: none;
}
.restore-confirm {
display: flex;
flex-direction: column;
gap: 0.35rem;
font-size: 0.875rem;
margin: 0 0 1rem;
}
</style>
+5 -52
View File
@@ -11,6 +11,7 @@ import PaginationBar from "@/components/PaginationBar.vue";
import MomentsSettings from "@/components/MomentsSettings.vue";
import ReplyShapesSettings from "@/components/ReplyShapesSettings.vue";
import BackupScheduleSettings from "@/components/BackupScheduleSettings.vue";
import RestoreBackupDialog from "@/components/RestoreBackupDialog.vue";
import { formatBytes } from "@/utils/bytes";
import { fmtDate, fmtLogStamp } from "@/utils/dateFormat";
import { fetchVersion, type VersionPayload } from "@/api/version";
@@ -467,7 +468,7 @@ const confirmNewPassword = ref("");
const changingPassword = ref(false);
const invalidatingSessions = ref(false);
const exporting = ref(false);
const restoring = ref(false);
const showRestore = ref(false);
// Backup, export and restore walk the whole store, so they are slow BY DESIGN
// and the client's ordinary 30s default would cut them off mid-work. They are
// still bounded — every wait has a deadline, not necessarily a short one — and
@@ -509,7 +510,6 @@ async function copyCommit() {
commitCopied.value = true;
setTimeout(() => { commitCopied.value = false; }, 2000);
}
const restoreFileInput = ref<HTMLInputElement | null>(null);
// Migrate stored "admin" → "config"; unknown tabs fall back to "general"
const VALID_TABS = new Set(["general", "account", "notifications", "integrations", "data", "apikeys", "config", "users", "logs", "groups", "areas", "platforms"]);
@@ -1079,10 +1079,6 @@ async function exportNotes(format: "markdown" | "json") {
}
}
function triggerRestoreUpload() {
restoreFileInput.value?.click();
}
async function saveNotifications() {
savingNotifications.value = true;
notificationsSaved.value = false;
@@ -1279,42 +1275,6 @@ async function runDbMaintenanceNow() {
}
}
async function handleRestoreFile(event: Event) {
const file = (event.target as HTMLInputElement).files?.[0];
if (!file) return;
restoring.value = true;
try {
// The scheduled backup's .tar goes up as it is; the JSON download is
// parsed first so a damaged file fails here, before the upload.
const isTar = file.name.toLowerCase().endsWith(".tar");
const res = await fetch("/api/admin/restore", {
method: "POST",
headers: { "Content-Type": isTar ? "application/x-tar" : "application/json" },
body: isTar ? file : JSON.stringify(JSON.parse(await file.text())),
signal: bulkDeadline(),
});
if (res.status === 413) {
throw new Error(
"the file is bigger than the web upload allows. Restore it with " +
"`python -m grimoire.restore <file>` inside the Grimoire container instead.",
);
}
if (!res.ok) {
const body = await res.json().catch(() => ({ error: `Error ${res.status}` }));
throw new Error((body as Record<string, string>).error || `Error ${res.status}`);
}
const result = await res.json();
toastStore.show(
`Restored ${result.stats?.users ?? 0} users, ${result.stats?.notes ?? 0} notes, ${result.stats?.conversations ?? 0} conversations`
);
} catch (e) {
toastStore.show("Restore failed: " + (e as Error).message, "error");
} finally {
restoring.value = false;
if (restoreFileInput.value) restoreFileInput.value.value = "";
}
}
async function testSearch() {
const q = searchQuery.value.trim();
if (!q) return;
@@ -2429,17 +2389,11 @@ async function deleteUser(userId: number) {
<button class="btn-secondary" @click="exportData('full')" :disabled="exporting">
{{ exporting ? "Exporting..." : "Full Backup" }}
</button>
<button class="btn-secondary btn-warn" @click="triggerRestoreUpload" :disabled="restoring">
{{ restoring ? "Restoring..." : "Restore from Backup" }}
<button class="btn-secondary btn-warn" @click="showRestore = true">
Restore from Backup
</button>
<input
ref="restoreFileInput"
type="file"
accept=".json,.tar"
class="hidden-file-input"
@change="handleRestoreFile"
/>
</div>
<RestoreBackupDialog v-if="showRestore" @close="showRestore = false" />
</section>
<section class="settings-section full-width">
<h2>Scheduled backups</h2>
@@ -3597,7 +3551,6 @@ async function deleteUser(userId: number) {
flex-wrap: wrap;
gap: 0.5rem;
}
.hidden-file-input { display: none; }
/* Checkboxes */
.checkbox-field { margin-bottom: 1rem; }
+100 -21
View File
@@ -17,7 +17,6 @@ from grimoire.services.auth import (
from grimoire.services.backup import (
export_full_backup,
export_user_backup,
restore_full_backup,
)
from grimoire.services.email import SMTP_SETTING_KEYS, get_base_url, get_smtp_config, is_smtp_configured, send_test_email
from grimoire.services.logging import get_logs, get_log_stats, log_audit
@@ -57,31 +56,111 @@ async def backup():
)
async def _restore_payload() -> tuple[dict | None, tuple | None]:
"""The backup a restore request names: `?source=<file>` at the backup
destination, or the uploaded file (the JSON download as
application/json, or the tar as the raw body; held to the request-size
limit, which is why the destination is the way in for a large one).
Returns (payload, None) or (None, an error response)."""
import io
import tempfile
from pathlib import Path
from grimoire.services import backup_destination as dest
from grimoire.services.backup_file import ArchiveError, read_archive
source = request.args.get("source")
try:
if source:
config = await dest.load_config()
secret, _ = await dest.load_secret()
with tempfile.TemporaryDirectory(prefix="grimoire-restore-") as staging:
path = await dest.open_backup(config, secret, source, Path(staging))
def load() -> dict:
with open(path, "rb") as f:
return read_archive(f)
data = await asyncio.to_thread(load)
elif request.mimetype == "application/json":
data = await request.get_json()
else:
data = read_archive(io.BytesIO(await request.get_data()))
except dest.DestinationError as exc:
return None, (jsonify({"error": str(exc)}), 400)
except ArchiveError as exc:
return None, (jsonify({"error": f"That file isn't a Grimoire backup: {exc}"}), 400)
if not isinstance(data, dict) or not data:
return None, (jsonify({"error": "No backup data provided"}), 400)
return data, None
@admin_bp.route("/restore/sources", methods=["GET"])
@admin_required
async def restore_sources():
"""The backups at the backup destination, newest first, to restore from
without uploading one."""
from grimoire.services import backup_destination as dest
config = await dest.load_config()
secret, _ = await dest.load_secret()
try:
backups = await asyncio.wait_for(dest.list_backups(config, secret), _BACKUP_CHECK_TIMEOUT)
except dest.DestinationError as exc:
return jsonify({"backups": [], "error": str(exc)})
except (TimeoutError, asyncio.TimeoutError):
return jsonify({"backups": [], "error": (
f"The destination didn't answer within {int(_BACKUP_CHECK_TIMEOUT)} seconds."
)})
return jsonify({"backups": [{"name": b.name, "size_bytes": b.size_bytes} for b in backups]})
@admin_bp.route("/restore/preview", methods=["POST"])
@admin_required
async def restore_preview():
"""What restoring this backup would do, changing nothing: fresh or
rollback, when it was taken, and the counts now and after. 409 with the
reason when it can't be restored here."""
from grimoire.services.backup import RestoreRefused, replace_preview
data, error = await _restore_payload()
if error:
return error
try:
return jsonify(await replace_preview(data))
except RestoreRefused as exc:
return jsonify({"error": str(exc)}), 409
@admin_bp.route("/restore", methods=["POST"])
@admin_required
async def restore():
"""Restore from an uploaded backup: the JSON download (application/json)
or the scheduled backup's tar (sent as the raw body). The upload is held
to the app's request-size limit; `python -m grimoire.restore` has none, and
is the way in for a bigger file or a fresh instance."""
if request.mimetype == "application/json":
data = await request.get_json()
else:
import io
"""Put the whole instance back to a full backup (services/backup.py
replace_from_backup), after backing up what it replaces. `?confirm=rollback`
is required: this removes everything made since the backup, for every
user. Every session ends, the caller's included; search is rebuilt in the
background."""
from grimoire.services import backup_job as job
from grimoire.services.backup import RestoreRefused
from grimoire.services.backup_file import ArchiveError, read_archive
try:
data = read_archive(io.BytesIO(await request.get_data()))
except ArchiveError as exc:
return jsonify({"error": f"That file isn't a Grimoire backup: {exc}"}), 400
if not data:
return jsonify({"error": "No backup data provided"}), 400
stats = await restore_full_backup(data)
if request.args.get("confirm") != "rollback":
return jsonify({"error": "Restoring replaces the whole instance; confirm it first."}), 400
data, error = await _restore_payload()
if error:
return error
try:
result = await job.rollback_to_backup(data)
except RestoreRefused as exc:
return jsonify({"error": str(exc)}), 409
except job.BackupBusy:
return jsonify({"error": "A backup is running. Restore once it has finished."}), 409
job.start_reindex_after_restore()
uid = get_current_user_id()
await log_audit("restore", user_id=uid, username=g.user.username, ip_address=client_ip(), details={"stats": stats})
return jsonify({"status": "ok", "stats": stats})
await log_audit(
"restore", user_id=uid, username=g.user.username, ip_address=client_ip(),
details={"mode": result["mode"], "exported_at": result.get("exported_at"),
"safety_backup": result["safety_backup"], "stats": result["restored"]},
)
return jsonify({"status": "ok", **result})
@admin_bp.route("/users", methods=["GET"])
+23
View File
@@ -2998,6 +2998,29 @@ async def check_replace(data: dict) -> str:
return await _replace_mode(session, data)
# What the restore preview counts, now and in the backup: the kinds a person
# recognises, not every table.
_PREVIEW_KINDS = (
("users", "Accounts"), ("projects", "Projects"), ("milestones", "Plans"),
("notes", "Notes and tasks"), ("rules", "Rules and preferences"),
)
async def replace_preview(data: dict) -> dict:
"""What replacing this store with `data` would do, changing nothing: the
mode, when the backup was taken, and per kind how many there are now and
how many there will be. RestoreRefused when it can't be applied here."""
from grimoire.models import Base
async with async_session() as session:
mode = await _replace_mode(session, data)
counts = []
for table, label in _PREVIEW_KINDS:
now = await _count(session, Base.metadata.tables[table])
counts.append({"kind": label, "now": now, "after": len(data.get(table) or [])})
return {"mode": mode, "exported_at": data.get("exported_at"), "counts": counts}
async def replace_from_backup(data: dict) -> dict:
"""Put this store back to a full backup, in one transaction.
+63 -3
View File
@@ -4,8 +4,9 @@ Two kinds of destination, both instance-wide and set by an admin:
- **folder**: a directory inside the container, usually a mounted share. A
backup service that watches the share (or a person) takes it from there.
- **s3**: any S3-compatible bucket, addressed by path. Grimoire only puts and
lists, so an append-only key is enough.
- **s3**: any S3-compatible bucket, addressed by path. Grimoire puts and
lists, and reads one back for a restore, so a key that can add and read
objects is enough; it never needs to delete.
Either way Grimoire ADDS one dated file per run and never removes one: keeping
and thinning the history is the destination's job (services/backup_file.py
@@ -28,7 +29,7 @@ from datetime import datetime
from pathlib import Path
from grimoire.services import backup_file
from grimoire.services.backup_s3 import S3Error, S3Target, list_objects, put_file
from grimoire.services.backup_s3 import S3Error, S3Target, get_file, list_objects, put_file
from grimoire.services.secrets import seal, unseal
from grimoire.services.settings import get_admin_setting, set_admin_setting
@@ -260,3 +261,62 @@ async def check_destination(
def _count_backups(folder: Path) -> int:
return sum(1 for name in os.listdir(folder) if backup_file.is_backup_name(name))
@dataclass(frozen=True)
class StoredBackup:
name: str
size_bytes: int
def _taken_order(name: str) -> str:
# The timestamp after the prefix, so grimoire- and scribe- names interleave
# by when they were taken.
return name.split("-", 1)[-1]
async def list_backups(config: BackupConfig, secret: str | None, limit: int = 50) -> list[StoredBackup]:
"""The backups at the destination, newest first."""
if config.destination == "folder":
folder = Path(config.folder_path)
if not folder.is_dir():
raise DestinationError(f"{folder} isn't there, or isn't a folder.")
def scan() -> list[StoredBackup]:
return [
StoredBackup(entry.name, entry.stat().st_size)
for entry in os.scandir(folder)
if entry.is_file() and backup_file.is_backup_name(entry.name)
]
found = await asyncio.to_thread(scan)
else:
problems = config.problems(has_secret=bool(secret))
if problems:
raise DestinationError(" ".join(problems))
try:
objects = await list_objects(s3_target(config, secret or ""))
except S3Error as exc:
raise DestinationError(str(exc)) from None
found = [StoredBackup(o.key, o.size) for o in objects if backup_file.is_backup_name(o.key)]
return sorted(found, key=lambda b: _taken_order(b.name), reverse=True)[:limit]
async def open_backup(config: BackupConfig, secret: str | None, name: str, staging: Path) -> Path:
"""A local path to the backup `name` at the destination: the file itself
in a folder, or a download into `staging` from a bucket. Only a name the
scheduled backup writes is accepted, so the request can't reach any other
file or object."""
if Path(name).name != name or not backup_file.is_backup_name(name):
raise DestinationError(f"{name!r} isn't the name of a Grimoire backup.")
if config.destination == "folder":
path = Path(config.folder_path) / name
if not path.is_file():
raise DestinationError(f"{name} isn't in {config.folder_path}.")
return path
target = s3_target(config, secret or "")
path = staging / name
try:
await get_file(target, name, path)
except S3Error as exc:
raise DestinationError(str(exc)) from None
return path
+9
View File
@@ -212,6 +212,15 @@ async def rollback_to_backup(data: dict) -> dict:
return result
def start_reindex_after_restore() -> None:
"""Rebuild the search vectors a rollback cleared, in the background."""
from grimoire.services.backup import reindex_after_restore
task = asyncio.get_running_loop().create_task(reindex_after_restore())
_TASKS.add(task)
task.add_done_callback(_TASKS.discard)
def start_backup_now() -> None:
"""Start a manual backup in the background (the request returns at once;
the page reads progress from `is_running` and the run list). Raises
+26 -3
View File
@@ -4,9 +4,10 @@ path-style addressing (endpoint/bucket/key), and the two calls a backup needs.
The signing is checked against AWS's published SigV4 examples in
tests/test_services_backup_s3.py.
Grimoire puts and lists; it never deletes. The bucket keeps the history and
thins it on its own policy, so a key that may only add objects is enough —
and is the safer key to hand a service, because a stolen one cannot wipe the
Grimoire puts and lists, and reads a backup back when an admin restores from
the bucket; it never deletes. The bucket keeps the history and thins it on
its own policy, so a key that may only add and read objects is enough — and
is the safer key to hand a service, because a stolen one cannot wipe the
history it writes into.
Written by hand rather than with boto3: two calls don't justify a large
@@ -171,6 +172,28 @@ async def put_file(target: S3Target, key: str, path: Path, *,
raise S3Error(f"Upload of {key} refused: {_says(response)}")
async def get_file(target: S3Target, key: str, path: Path, *,
transport: httpx.AsyncBaseTransport | None = None,
now: datetime | None = None) -> None:
"""Download one object to `path`, streamed to disk."""
url = target.url(key)
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
target.region, now or datetime.now(timezone.utc))
try:
async with httpx.AsyncClient(verify=target.verify_tls,
timeout=UPLOAD_TIMEOUT_SECONDS,
transport=transport) as client:
async with client.stream("GET", url, headers=headers) as response:
if response.status_code != 200:
await response.aread()
raise S3Error(f"Reading {key} refused: {_says(response)}")
with open(path, "wb") as f:
async for chunk in response.aiter_bytes(CHUNK):
f.write(chunk)
except httpx.HTTPError as exc:
raise _reach_error(target, exc) from None
async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
transport: httpx.AsyncBaseTransport | None = None,
now: datetime | None = None) -> list[StoredObject]:
+6
View File
@@ -52,3 +52,9 @@ def test_service_surface_and_allowlist():
def test_set_admin_setting_exists():
from grimoire.services.settings import set_admin_setting
assert callable(set_admin_setting)
def test_restore_routes_registered():
from grimoire.app import create_app
rules = {r.rule for r in create_app().url_map.iter_rules()}
assert {"/api/admin/restore", "/api/admin/restore/preview", "/api/admin/restore/sources"} <= rules
+44
View File
@@ -188,3 +188,47 @@ def test_checking_s3_only_lists():
said = asyncio.run(check_destination(S3, "secret", transport=httpx.MockTransport(handler)))
assert methods == ["GET", "GET"]
assert "holds 2 Grimoire backup" in said
# ── Restoring from what is already at the destination ──────────────────────
def test_a_folder_lists_its_backups_newest_first_and_nothing_else(tmp_path):
for name in ("grimoire-20261012T020000Z.tar", "scribe-20261011T020000Z.tar",
"grimoire-20261013T020000Z.tar", "notes.txt", ".grimoire-20261014T020000Z.tar.part"):
(tmp_path / name).write_bytes(b"x" * 10)
found = asyncio.run(backup_destination.list_backups(BackupConfig(folder_path=str(tmp_path)), None))
assert [b.name for b in found] == [
"grimoire-20261013T020000Z.tar", "grimoire-20261012T020000Z.tar",
"scribe-20261011T020000Z.tar",
]
assert found[0].size_bytes == 10
def test_a_missing_folder_says_so_when_listing(tmp_path):
with pytest.raises(DestinationError, match="isn't there"):
asyncio.run(backup_destination.list_backups(
BackupConfig(folder_path=str(tmp_path / "absent")), None))
def test_opening_a_folder_backup_is_the_file_itself(tmp_path):
(tmp_path / "grimoire-20261012T020000Z.tar").write_bytes(b"x")
path = asyncio.run(backup_destination.open_backup(
BackupConfig(folder_path=str(tmp_path)), None, "grimoire-20261012T020000Z.tar", tmp_path))
assert path == tmp_path / "grimoire-20261012T020000Z.tar"
@pytest.mark.parametrize("name", [
"../etc/passwd", "grimoire-x/../../secret.tar", "secret_key", "/data/secret_key",
"grimoire-20261012T020000Z.json",
])
def test_only_a_backup_name_can_be_opened(tmp_path, name):
with pytest.raises(DestinationError, match="isn't the name of a Grimoire backup"):
asyncio.run(backup_destination.open_backup(
BackupConfig(folder_path=str(tmp_path)), None, name, tmp_path))
def test_a_backup_not_in_the_folder_says_so(tmp_path):
with pytest.raises(DestinationError, match="isn't in"):
asyncio.run(backup_destination.open_backup(
BackupConfig(folder_path=str(tmp_path)), None, "grimoire-20261012T020000Z.tar", tmp_path))
+24
View File
@@ -126,3 +126,27 @@ def test_list_reads_contents():
s3.StoredObject("grimoire-20261011T000000Z.tar", 1200, "2026-10-11T00:00:01.000Z"),
s3.StoredObject("grimoire-20261012T000000Z.tar", 1300, "2026-10-12T00:00:01.000Z"),
]
def test_get_streams_the_object_to_disk(tmp_path):
seen = {}
def handler(request):
seen["method"], seen["url"] = request.method, str(request.url)
seen["sha"] = request.headers["x-amz-content-sha256"]
return httpx.Response(200, content=b"y" * 3000)
out = tmp_path / "b.tar"
asyncio.run(s3.get_file(TARGET, "b.tar", out, transport=httpx.MockTransport(handler)))
assert seen["method"] == "GET"
assert seen["url"] == "http://archives:8081/s3/grimoire/b.tar"
assert seen["sha"] == s3.EMPTY_SHA256
assert out.read_bytes() == b"y" * 3000
def test_a_refused_get_says_so_and_writes_nothing(tmp_path):
transport = httpx.MockTransport(lambda r: httpx.Response(403, content=b"<Error><Code>AccessDenied</Code></Error>"))
out = tmp_path / "b.tar"
with pytest.raises(s3.S3Error, match="b.tar"):
asyncio.run(s3.get_file(TARGET, "b.tar", out, transport=transport))
assert not out.exists()