feat(6848): Settings restore rolls back, from the destination or a file, after a preview
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / Python lint (push) Successful in 1s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 1m42s
CI & Build / integration (push) Successful in 1m48s
CI & Build / Build & push image (push) Canceled after 22s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / Python lint (push) Successful in 1s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 1m42s
CI & Build / integration (push) Successful in 1m48s
CI & Build / Build & push image (push) Canceled after 22s
The web restore no longer copies a backup in beside the live data. - GET /api/admin/restore/sources lists the backups at the backup destination, newest first. - POST /api/admin/restore/preview says what a restore would do, changing nothing: fresh or rollback, when the backup was taken, and counts now and after per kind. A backup that cannot apply here gets a 409 with the reason. - POST /api/admin/restore?confirm=rollback runs backup_job.rollback_to_backup: safety backup, replace, then search rebuilt in the background. Each takes ?source=<name> (a backup at the destination) or an upload. The destination path exists because uploads are held to the request-size limit, and a real instance's backup is far over it. backup_s3.get_file downloads from a bucket. backup_destination.list_backups/open_backup only accept names the scheduled backup writes, so a request cannot reach any other file. Settings -> Data -> Restore from Backup opens RestoreBackupDialog. The dialog: 1. Pick a backup at the destination or a file. 2. Read the preview. 3. Type "roll back". 4. Run it. The dialog then says where the replaced state was saved, and sends the user to sign in again. Closes #6840. Milestone 582, step 4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2998,6 +2998,29 @@ async def check_replace(data: dict) -> str:
|
||||
return await _replace_mode(session, data)
|
||||
|
||||
|
||||
# What the restore preview counts, now and in the backup: the kinds a person
|
||||
# recognises, not every table.
|
||||
_PREVIEW_KINDS = (
|
||||
("users", "Accounts"), ("projects", "Projects"), ("milestones", "Plans"),
|
||||
("notes", "Notes and tasks"), ("rules", "Rules and preferences"),
|
||||
)
|
||||
|
||||
|
||||
async def replace_preview(data: dict) -> dict:
|
||||
"""What replacing this store with `data` would do, changing nothing: the
|
||||
mode, when the backup was taken, and per kind how many there are now and
|
||||
how many there will be. RestoreRefused when it can't be applied here."""
|
||||
from grimoire.models import Base
|
||||
|
||||
async with async_session() as session:
|
||||
mode = await _replace_mode(session, data)
|
||||
counts = []
|
||||
for table, label in _PREVIEW_KINDS:
|
||||
now = await _count(session, Base.metadata.tables[table])
|
||||
counts.append({"kind": label, "now": now, "after": len(data.get(table) or [])})
|
||||
return {"mode": mode, "exported_at": data.get("exported_at"), "counts": counts}
|
||||
|
||||
|
||||
async def replace_from_backup(data: dict) -> dict:
|
||||
"""Put this store back to a full backup, in one transaction.
|
||||
|
||||
|
||||
@@ -4,8 +4,9 @@ Two kinds of destination, both instance-wide and set by an admin:
|
||||
|
||||
- **folder**: a directory inside the container, usually a mounted share. A
|
||||
backup service that watches the share (or a person) takes it from there.
|
||||
- **s3**: any S3-compatible bucket, addressed by path. Grimoire only puts and
|
||||
lists, so an append-only key is enough.
|
||||
- **s3**: any S3-compatible bucket, addressed by path. Grimoire puts and
|
||||
lists, and reads one back for a restore, so a key that can add and read
|
||||
objects is enough; it never needs to delete.
|
||||
|
||||
Either way Grimoire ADDS one dated file per run and never removes one: keeping
|
||||
and thinning the history is the destination's job (services/backup_file.py
|
||||
@@ -28,7 +29,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from grimoire.services import backup_file
|
||||
from grimoire.services.backup_s3 import S3Error, S3Target, list_objects, put_file
|
||||
from grimoire.services.backup_s3 import S3Error, S3Target, get_file, list_objects, put_file
|
||||
from grimoire.services.secrets import seal, unseal
|
||||
from grimoire.services.settings import get_admin_setting, set_admin_setting
|
||||
|
||||
@@ -260,3 +261,62 @@ async def check_destination(
|
||||
|
||||
def _count_backups(folder: Path) -> int:
|
||||
return sum(1 for name in os.listdir(folder) if backup_file.is_backup_name(name))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredBackup:
|
||||
name: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
def _taken_order(name: str) -> str:
|
||||
# The timestamp after the prefix, so grimoire- and scribe- names interleave
|
||||
# by when they were taken.
|
||||
return name.split("-", 1)[-1]
|
||||
|
||||
|
||||
async def list_backups(config: BackupConfig, secret: str | None, limit: int = 50) -> list[StoredBackup]:
|
||||
"""The backups at the destination, newest first."""
|
||||
if config.destination == "folder":
|
||||
folder = Path(config.folder_path)
|
||||
if not folder.is_dir():
|
||||
raise DestinationError(f"{folder} isn't there, or isn't a folder.")
|
||||
|
||||
def scan() -> list[StoredBackup]:
|
||||
return [
|
||||
StoredBackup(entry.name, entry.stat().st_size)
|
||||
for entry in os.scandir(folder)
|
||||
if entry.is_file() and backup_file.is_backup_name(entry.name)
|
||||
]
|
||||
found = await asyncio.to_thread(scan)
|
||||
else:
|
||||
problems = config.problems(has_secret=bool(secret))
|
||||
if problems:
|
||||
raise DestinationError(" ".join(problems))
|
||||
try:
|
||||
objects = await list_objects(s3_target(config, secret or ""))
|
||||
except S3Error as exc:
|
||||
raise DestinationError(str(exc)) from None
|
||||
found = [StoredBackup(o.key, o.size) for o in objects if backup_file.is_backup_name(o.key)]
|
||||
return sorted(found, key=lambda b: _taken_order(b.name), reverse=True)[:limit]
|
||||
|
||||
|
||||
async def open_backup(config: BackupConfig, secret: str | None, name: str, staging: Path) -> Path:
|
||||
"""A local path to the backup `name` at the destination: the file itself
|
||||
in a folder, or a download into `staging` from a bucket. Only a name the
|
||||
scheduled backup writes is accepted, so the request can't reach any other
|
||||
file or object."""
|
||||
if Path(name).name != name or not backup_file.is_backup_name(name):
|
||||
raise DestinationError(f"{name!r} isn't the name of a Grimoire backup.")
|
||||
if config.destination == "folder":
|
||||
path = Path(config.folder_path) / name
|
||||
if not path.is_file():
|
||||
raise DestinationError(f"{name} isn't in {config.folder_path}.")
|
||||
return path
|
||||
target = s3_target(config, secret or "")
|
||||
path = staging / name
|
||||
try:
|
||||
await get_file(target, name, path)
|
||||
except S3Error as exc:
|
||||
raise DestinationError(str(exc)) from None
|
||||
return path
|
||||
|
||||
@@ -212,6 +212,15 @@ async def rollback_to_backup(data: dict) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
def start_reindex_after_restore() -> None:
|
||||
"""Rebuild the search vectors a rollback cleared, in the background."""
|
||||
from grimoire.services.backup import reindex_after_restore
|
||||
|
||||
task = asyncio.get_running_loop().create_task(reindex_after_restore())
|
||||
_TASKS.add(task)
|
||||
task.add_done_callback(_TASKS.discard)
|
||||
|
||||
|
||||
def start_backup_now() -> None:
|
||||
"""Start a manual backup in the background (the request returns at once;
|
||||
the page reads progress from `is_running` and the run list). Raises
|
||||
|
||||
@@ -4,9 +4,10 @@ path-style addressing (endpoint/bucket/key), and the two calls a backup needs.
|
||||
The signing is checked against AWS's published SigV4 examples in
|
||||
tests/test_services_backup_s3.py.
|
||||
|
||||
Grimoire puts and lists; it never deletes. The bucket keeps the history and
|
||||
thins it on its own policy, so a key that may only add objects is enough —
|
||||
and is the safer key to hand a service, because a stolen one cannot wipe the
|
||||
Grimoire puts and lists, and reads a backup back when an admin restores from
|
||||
the bucket; it never deletes. The bucket keeps the history and thins it on
|
||||
its own policy, so a key that may only add and read objects is enough — and
|
||||
is the safer key to hand a service, because a stolen one cannot wipe the
|
||||
history it writes into.
|
||||
|
||||
Written by hand rather than with boto3: two calls don't justify a large
|
||||
@@ -171,6 +172,28 @@ async def put_file(target: S3Target, key: str, path: Path, *,
|
||||
raise S3Error(f"Upload of {key} refused: {_says(response)}")
|
||||
|
||||
|
||||
async def get_file(target: S3Target, key: str, path: Path, *,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
now: datetime | None = None) -> None:
|
||||
"""Download one object to `path`, streamed to disk."""
|
||||
url = target.url(key)
|
||||
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
|
||||
target.region, now or datetime.now(timezone.utc))
|
||||
try:
|
||||
async with httpx.AsyncClient(verify=target.verify_tls,
|
||||
timeout=UPLOAD_TIMEOUT_SECONDS,
|
||||
transport=transport) as client:
|
||||
async with client.stream("GET", url, headers=headers) as response:
|
||||
if response.status_code != 200:
|
||||
await response.aread()
|
||||
raise S3Error(f"Reading {key} refused: {_says(response)}")
|
||||
with open(path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(CHUNK):
|
||||
f.write(chunk)
|
||||
except httpx.HTTPError as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
|
||||
|
||||
async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
now: datetime | None = None) -> list[StoredObject]:
|
||||
|
||||
Reference in New Issue
Block a user