diff --git a/steward/app.py b/steward/app.py index 19dc84e..bd14640 100644 --- a/steward/app.py +++ b/steward/app.py @@ -175,6 +175,10 @@ def create_app( app.jinja_env.globals["csrf_field"] = _csrf.csrf_field # The sign-in page offers "Forgot your password?" only while mail can send (milestone 497). app.jinja_env.globals["reset_by_email"] = lambda: bool(app.config.get("SMTP", {}).get("host")) + # While a backup is being restored nothing else reads or writes the + # database: a page says so, and agents get a 503 to retry (#5816). + from .backup import restoring as _restoring + app.before_request(_restoring.gate) app.before_request(_csrf.check_request) # The session cookie is Secure, and HSTS is sent, only on requests that # arrived over HTTPS — directly or as a trusted proxy reports it @@ -595,5 +599,6 @@ async def _mark_interrupted_runs(app: Quart) -> None: async def _run_scheduler(app: Quart) -> None: + from .backup.restoring import is_restoring from .core.scheduler import start_scheduler - await start_scheduler(app._task_registry) + await start_scheduler(app._task_registry, paused=is_restoring) diff --git a/steward/audit/service.py b/steward/audit/service.py index e7683b8..3e61c8d 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -200,6 +200,8 @@ ACTIONS = { "prometheus.template_deleted": "deleted Prometheus widget template", "backup.made": "sent a backup", "backup.failed": "tried to send a backup, which failed:", + "backup.restored": "restored Steward from a backup", + "backup.restore_failed": "tried to restore Steward from a backup, which failed:", } # The filter's choices: an area, matched as a prefix of the action code. diff --git a/steward/auth/csrf.py b/steward/auth/csrf.py index 561ea1d..d6d8b90 100644 --- a/steward/auth/csrf.py +++ b/steward/auth/csrf.py @@ -52,6 +52,7 @@ def rotate() -> None: # a backup file (settings only is small; an older one carries the audit log). UPLOAD_LIMITS: dict[str, int] = { "/settings/backups/check/file/": 512 * 1024 * 1024, + "/settings/backups/restore/file/": 512 * 1024 * 1024, } diff --git a/steward/backup/restoring.py b/steward/backup/restoring.py new file mode 100644 index 0000000..09efb79 --- /dev/null +++ b/steward/backup/restoring.py @@ -0,0 +1,272 @@ +"""Restoring mode (#5816): what a running Steward does while a backup goes back +in, from Settings → Backups. + +1. Read the backup and refuse it up front if it can't go in: not a backup, + or a different secret key while STEWARD_SECRET_KEY pins this one. +2. Take a safety backup of Steward as it stands: kept on /data/backups and + sent to the bucket when one is set. A restore can then be undone the same + way. Without it nothing goes further. +3. While it runs, Steward is restoring: the scheduler holds its ticks, agent + ingest answers 503 (agents keep their samples and send them again), and + every page says what is happening. +4. In place for a backup of this schema (restore.restore_in_place), or a + full replace the person agreed to for an older one. +5. The backup's key is put in place, the restore audited, and the process + exits with RESTART_EXIT_CODE. Swarm, compose or systemd starts it again, + and settings, the key and plugins load fresh. The person who restored + signs in again: users came back from the backup. + +A restore that fails in place has changed nothing; Steward carries on and +says why. A full replace that fails part-way leaves an empty database, so +Steward restarts into first-run setup, where the safety backup can be +uploaded. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import secrets +import shutil +import signal +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger(__name__) + +SAFETY_DIR = Path("/data/backups") +# Safety backups kept on /data, newest first; older ones are on the bucket. +SAFETY_KEEP = 5 +# Not 0: systemd's Restart=on-failure restarts only a failed exit. +RESTART_EXIT_CODE = 3 +# Long enough for the restoring page's next poll to say it is done. +EXIT_DELAY_SECONDS = 5 +# A backup chosen for restore waits here between its check and the +# confirmation (#5817); one not confirmed within the hour is removed. +STAGED_SECONDS = 3600 + + +@dataclass +class Restoring: + active: bool = False + name: str = "" + by: str = "" + since: datetime | None = None + step: str = "" + error: str = "" + restart_requested: bool = False + + +STATE = Restoring() +_tasks: set[asyncio.Task] = set() + + +def is_restoring() -> bool: + return STATE.active + + +def start_restore(app, path: Path, name: str, *, username: str, replace_all: bool = False, + owned_dir: Path | None = None) -> bool: + """Start restoring `path` in the background; False if a restore is already + under way. `owned_dir`, when given, is removed once it is over.""" + if STATE.active or STATE.restart_requested: + return False + STATE.active, STATE.name, STATE.by = True, name, username + STATE.since, STATE.step, STATE.error = datetime.now(timezone.utc), "Reading the backup", "" + task = asyncio.get_running_loop().create_task( + _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir)) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return True + + +async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, + owned_dir: Path | None) -> None: + from steward.core.audit import log_audit + + from .dump import BackupError + from .s3 import S3Error + wiped = False + try: + key = await asyncio.to_thread(_backup_key, path) + pinned = os.environ.get("STEWARD_SECRET_KEY") + if pinned and pinned != key: + raise BackupError("STEWARD_SECRET_KEY is set and differs from the backup's key; " + "set it to the backup's key first, or its passwords and keys can't be read.") + STATE.step = "Saving a safety backup of Steward as it stands" + safety = await _safety_backup(app) + STATE.step = "Restoring" + plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] + if replace_all: + from .restore import restore + wiped = True + done = await restore(path, app.config["DATABASE_URL"], wipe=True, plugin_dirs=plugin_dirs) + detail = {"mode": "replaced everything"} + else: + from .restore import restore_in_place + done = await restore_in_place(path, app.config["DATABASE_URL"], plugin_dirs=plugin_dirs) + detail = {"mode": "in place", "tables": len(done.tables), "rows": done.rows, + "put_right": done.fixed} + if done.secret_key != app.config["SECRET_KEY"] and not pinned: + _write_key(done.secret_key) + await log_audit(app, None, username, "backup.restored", entity_type="backup", + entity_id=name, detail=detail | {"safety_backup": safety}) + logger.warning("Restored %s (%s); restarting", name, detail["mode"]) + STATE.step = "Restored. Steward is restarting; sign in again when it is back" + _restart_soon() + except (BackupError, S3Error, OSError) as exc: + _failed(app, name, username, str(exc), wiped) + except Exception as exc: + logger.exception("Restore of %s failed", name) + _failed(app, name, username, f"Unexpected error: {exc}", wiped) + finally: + if owned_dir is not None: + shutil.rmtree(owned_dir, ignore_errors=True) + + +def _failed(app, name: str, username: str, error: str, wiped: bool) -> None: + from steward.core.audit import log_audit + logger.error("Restore of %s failed: %s", name, error) + STATE.error = error + if wiped: + # The database was dropped before the restore failed: start again + # into first-run setup, where the safety backup can be uploaded. + STATE.step = ("The restore failed after the database was emptied; Steward is restarting " + "into first-run setup, where the safety backup can be uploaded") + _restart_soon() + return + STATE.active = False + task = asyncio.get_running_loop().create_task(log_audit( + app, None, username, "backup.restore_failed", entity_type="backup", entity_id=name, + detail={"error": error})) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + + +def _backup_key(path: Path) -> str: + import tarfile + + from .dump import KEY_NAME, BackupError + try: + with tarfile.open(path, "r") as tar: + return tar.extractfile(KEY_NAME).read().decode().strip() + except (tarfile.TarError, OSError, KeyError, AttributeError): + raise BackupError(f"{path.name} isn't a Steward backup.") from None + + +async def _safety_backup(app) -> str: + """A backup of Steward as it stands, before anything is replaced: on + /data/backups always, and on the bucket when one is set. Its run is in + Recent backups like any other.""" + from steward.core.settings import get_all_settings + + from .dump import make_backup + from .job import _record, migration_heads, target_from + from .s3 import put_file + SAFETY_DIR.mkdir(parents=True, exist_ok=True) + url = app.config["DATABASE_URL"] + made = await make_backup(url, app.config["SECRET_KEY"], await migration_heads(url), SAFETY_DIR) + async with app.db_sessionmaker() as db: + target = target_from(await get_all_settings(db)) + entry = {"at": datetime.now(timezone.utc).isoformat(), "ok": True, "by": "before restore", + "name": made.name, "size": made.size, "secs": 0.0, "error": ""} + if target is not None: + await put_file(target, made.name, made.path) + else: + entry["error"] = "Kept on /data/backups only: no bucket is set up." + await _record(app, entry, None, "") + _prune() + return made.name + + +def _prune() -> None: + kept = sorted(SAFETY_DIR.glob("steward-*.tar"), reverse=True) + for old in kept[SAFETY_KEEP:]: + old.unlink(missing_ok=True) + + +def _write_key(key: str) -> None: + from steward.config import _SECRET_KEY_FILE + _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True) + _SECRET_KEY_FILE.write_text(key) + _SECRET_KEY_FILE.chmod(0o600) + + +def _restart_soon() -> None: + STATE.restart_requested = True + loop = asyncio.get_running_loop() + loop.call_later(EXIT_DELAY_SECONDS, os.kill, os.getpid(), signal.SIGTERM) + + +# ── Chosen, checked, waiting for a yes (#5817) ───────────────────────────── + +@dataclass +class Staged: + token: str + dir: Path + name: str = "" + schema: str = "" + ok: bool = False + + @property + def path(self) -> Path: + return self.dir / "backup.tar" + + def save(self) -> None: + (self.dir / "staged.json").write_text(json.dumps( + {"name": self.name, "schema": self.schema, "ok": self.ok})) + + +def _staging() -> Path: + return SAFETY_DIR / "incoming" + + +def stage() -> Staged: + """A fresh place to hold one backup until its restore is confirmed. Old + ones not confirmed are removed first.""" + root = _staging() + root.mkdir(parents=True, exist_ok=True) + cutoff = time.time() - STAGED_SECONDS + for old in root.iterdir(): + if old.stat().st_mtime < cutoff: + shutil.rmtree(old, ignore_errors=True) + token = secrets.token_hex(16) + (root / token).mkdir(mode=0o700) + return Staged(token, root / token) + + +def staged(token: str) -> Staged | None: + """The backup held under `token`, or None when there is none (or it was + removed for its age).""" + if not re.fullmatch(r"[0-9a-f]{32}", token or ""): + return None + d = _staging() / token + try: + meta = json.loads((d / "staged.json").read_text()) + except (OSError, ValueError): + return None + if not (d / "backup.tar").is_file(): + return None + return Staged(token, d, meta.get("name", ""), meta.get("schema", ""), bool(meta.get("ok"))) + + +# ── While it runs ─────────────────────────────────────────────────────────── + +async def gate(): + """before_request: while restoring, nothing reads or writes the database. + Agents and scripts get a 503 to retry; a page says what is happening.""" + from quart import Response, g, render_template, request + if not (STATE.active or STATE.restart_requested) or request.path.startswith("/static/"): + return None + g.database_down = True # the page's own reads (the sidebar's alert count) wait + headers = {"Retry-After": "30"} + wants_page = "text/html" in request.headers.get("Accept", "") + if request.method != "GET" or request.headers.get("HX-Request") or not wants_page: + return Response("Steward is restoring a backup; try again shortly.", status=503, + headers=headers, content_type="text/plain; charset=utf-8") + page = await render_template("errors/restoring.html", state=STATE) + return Response(page, status=503, headers=headers) diff --git a/steward/cli.py b/steward/cli.py index bfc098a..6ed361f 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -58,6 +58,11 @@ def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) -> app.run(host=host, port=port, debug=True) return asyncio.run(_serve(app, hypercorn_config(host, port))) + from .backup.restoring import RESTART_EXIT_CODE, STATE as restoring + if restoring.restart_requested: + # A restore finished: exit as failed so every supervisor starts + # Steward again (systemd's Restart=on-failure ignores a clean exit). + raise SystemExit(RESTART_EXIT_CODE) @main.command("backup") diff --git a/steward/core/scheduler.py b/steward/core/scheduler.py index 9379304..5245258 100644 --- a/steward/core/scheduler.py +++ b/steward/core/scheduler.py @@ -60,8 +60,10 @@ class _DueTracker: self.in_flight.discard(name) -async def start_scheduler(tasks: list[ScheduledTask]) -> None: - """Run scheduled tasks in a loop. Call with asyncio.create_task().""" +async def start_scheduler(tasks: list[ScheduledTask], paused=None) -> None: + """Run scheduled tasks in a loop. Call with asyncio.create_task(). + `paused`, when given, is asked each tick: while it says so nothing new + starts (a restore is under way, steward/backup/restoring.py).""" tracker = _DueTracker() def _spawn(task: ScheduledTask, now: float) -> None: @@ -77,8 +79,9 @@ async def start_scheduler(tasks: list[ScheduledTask]) -> None: while True: now = asyncio.get_event_loop().time() - for task in tracker.due(tasks, now): - _spawn(task, now) + if paused is None or not paused(): + for task in tracker.due(tasks, now): + _spawn(task, now) await asyncio.sleep(1) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index c029f4d..af51bfb 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -695,7 +695,7 @@ async def send_report_now(): async def _backups_page(flash=None): """Settings → Backups (#5673): how the newest backup went, the bucket and the daily hour (in the reader's zone), and the runs before.""" - from steward.backup import job + from steward.backup import job, restoring from steward.core.formatting import zone async with current_app.db_sessionmaker() as db: settings = await get_all_settings(db) @@ -705,7 +705,8 @@ async def _backups_page(flash=None): "settings/backups.html", settings=settings, flash=flash, shown_hour=hour, verdict=job.verdict(settings, job.is_running()), history=job.history_rows(settings.get("backup.history")), - has_secret=bool(settings.get("backup.s3_secret_key"))) + has_secret=bool(settings.get("backup.s3_secret_key")), + restore_failed=restoring.STATE if restoring.STATE.error else None) @settings_bp.get("/backups/") @@ -824,6 +825,108 @@ async def check_uploaded_backup(): return await _check_page(report=report, name=name) + +# Restore (#5817): choose a backup, see what it would change (the same check as +# above), then type to confirm. The chosen file is held server-side between +# the two (restoring.stage), so the confirmation names it by a token. + +RESTORE_WORD = "restore" + + +async def _restore_page(report=None, error: str = "", name: str = "", held=None): + return await render_template("settings/backup_restore.html", report=report, error=error, + name=name, held=held, word=RESTORE_WORD) + + +async def _hold_and_check(held, name: str): + from steward.backup import job + from steward.backup.dump import BackupError + try: + report = await job.check_file(current_app._get_current_object(), held.path, name) + except BackupError as exc: + import shutil + shutil.rmtree(held.dir, ignore_errors=True) + return await _restore_page(error=str(exc), name=name) + held.name, held.schema, held.ok = name, report.schema, report.ok + held.save() + return await _restore_page(report=report, name=name, held=held) + + +@settings_bp.post("/backups/restore/") +@require_role(UserRole.admin) +async def restore_stored_backup(): + """Fetch one of the bucket's backups, check it, and ask to confirm.""" + import shutil + + from steward.backup import job, restoring + from steward.backup.dump import BackupError + from steward.backup.s3 import S3Error, get_file + name = (await request.form).get("name", "") + if not job.is_backup_name(name): + return await _restore_page(error=f"{name!r} isn't the name of a Steward backup.", name=name) + async with current_app.db_sessionmaker() as db: + target = job.target_from(await get_all_settings(db)) + if target is None: + return await _restore_page(error="The bucket isn't set up: fill in the endpoint, bucket and keys.", + name=name) + held = restoring.stage() + try: + await get_file(target, name, held.path) + except (BackupError, S3Error) as exc: + shutil.rmtree(held.dir, ignore_errors=True) + error = str(exc) + if "AccessDenied" in error: + error += (" The bucket's key may not read backups back; download the file from the " + "bucket and restore it with Restore a file instead.") + return await _restore_page(error=error, name=name) + return await _hold_and_check(held, name) + + +@settings_bp.post("/backups/restore/file/") +@require_role(UserRole.admin) +async def restore_uploaded_backup(): + """A backup file sent from the browser, checked and held for the + confirmation. Its size limit is raised in auth/csrf.py UPLOAD_LIMITS.""" + from steward.backup import restoring + upload = (await request.files).get("file") + if upload is None or not upload.filename: + return await _restore_page(error="Choose a backup file to restore.") + held = restoring.stage() + await upload.save(held.path) + return await _hold_and_check(held, Path(upload.filename).name) + + +@settings_bp.post("/backups/restore/start/") +@require_role(UserRole.admin) +async def start_restore(): + """The confirmation: the held backup goes back in (restoring.py). Every + page shows the restore until Steward restarts.""" + import shutil + + from steward.backup import job, restoring + form = await request.form + held = restoring.staged(form.get("token", "")) + if held is None: + return await _restore_page(error="That backup is no longer held (it is removed an hour after " + "it was checked). Choose it again.") + if held.schema == "newer" or not held.ok: + shutil.rmtree(held.dir, ignore_errors=True) + return await _restore_page(error="This backup can't be restored here; check it again to see why.", + name=held.name) + replace_all = held.schema == "older" + if form.get("confirm", "").strip().lower() != RESTORE_WORD or ( + replace_all and "replace_all" not in form): + report = await job.check_file(current_app._get_current_object(), held.path, held.name) + return await _restore_page(report=report, name=held.name, held=held, + error=f"Type {RESTORE_WORD} to confirm" + + (", and tick Replace everything." if replace_all else ".")) + started = restoring.start_restore(current_app._get_current_object(), held.path, held.name, + username=session.get("username", "") or "an admin", + replace_all=replace_all, owned_dir=held.dir) + if not started: + return await _restore_page(error="A restore is already under way.", name=held.name) + return redirect(url_for("settings.backups")) + def _build_plugin_cfg_from_form(plugin: dict, form) -> dict: """Extract config values for one plugin from a form submission.""" name = plugin["_dir"] diff --git a/steward/templates/errors/restoring.html b/steward/templates/errors/restoring.html new file mode 100644 index 0000000..12e6b4d --- /dev/null +++ b/steward/templates/errors/restoring.html @@ -0,0 +1,18 @@ +{# errors/restoring.html — every page while a backup is being restored (#5816): + what is going back in, by whom, the step it is on. It reloads itself, so + the sign-in page appears once Steward is back. #} +{% extends "base.html" %} +{% block title %}Restoring a backup — Steward{% endblock %} +{% block head %}{% endblock %} +{% block content %} +
{{ state.name }}{% if state.by %}, started by {{ state.by }}{% endif %} + {%- if state.since %} {{ state.since|ago }}{% endif %}.
+{{ state.step }}…
++ Nothing else reads or writes Steward's records until it is done. Agents keep their readings + and send them when Steward is back. This page reloads by itself. +
+{{ report.summary() }}
+ +| Area | Same | Differ | Only in the backup | Only here |
|---|---|---|---|---|
| {{ a.name }} | +{{ a.same }} | +{% if differ %}{{ differ }}{% else %}—{% endif %} | +{% if only_b %}{{ only_b }}{% else %}—{% endif %} | +{% if only_h %}{{ only_h }}{% else %}—{% endif %} | +
{{ d.backup_value if d.backup_value is not none else "nothing" }}, here it is {{ d.here_value if d.here_value is not none else "nothing" }}.
+ {%- else %} in {{ d.columns|join(", ") }}.{% endif %}Left out of the comparison: what Steward records as it runs + (when an agent or monitor was last seen, a schedule's last run, the backup history), since it changes by the + minute. A restore still brings those back as they were in the backup.
diff --git a/steward/templates/settings/backup_check.html b/steward/templates/settings/backup_check.html index f4ef323..64be378 100644 --- a/steward/templates/settings/backup_check.html +++ b/steward/templates/settings/backup_check.html @@ -11,64 +11,7 @@ {% if error %}{{ name or "The backup" }} couldn't be checked. {{ error }}
{% else %} -{% set tone = "tone-crit" if not report.ok else ("tone-ok" if not report.changes else "tone-warn") %} -{{ report.summary() }}
- -| Area | Same | Differ | Only in the backup | Only here |
|---|---|---|---|---|
| {{ a.name }} | -{{ a.same }} | -{% if differ %}{{ differ }}{% else %}—{% endif %} | -{% if only_b %}{{ only_b }}{% else %}—{% endif %} | -{% if only_h %}{{ only_h }}{% else %}—{% endif %} | -
{{ d.backup_value if d.backup_value is not none else "nothing" }}, here it is {{ d.here_value if d.here_value is not none else "nothing" }}.
- {%- else %} in {{ d.columns|join(", ") }}.{% endif %}Left out of the comparison: what Steward records as it runs - (when an agent or monitor was last seen, a schedule's last run, the backup history), since it changes by the - minute. A restore still brings those back as they were in the backup.
+{% include "settings/_backup_report.html" %} {% endif %} {% endblock %} diff --git a/steward/templates/settings/backup_restore.html b/steward/templates/settings/backup_restore.html new file mode 100644 index 0000000..69f27ee --- /dev/null +++ b/steward/templates/settings/backup_restore.html @@ -0,0 +1,58 @@ +{# settings/backup_restore.html — restoring a backup, before anything changes + (#5817): what it would bring back (the same check as Check, #5823), what a + restore does, then a typed confirmation. A backup that can't go in says + why and offers no button. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% from "settings/_tabs.html" import settings_tabs %} +{% block title %}Settings — Backups — Restore — Steward{% endblock %} +{% block section_tabs %}{{ settings_tabs("backups") }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Backups", "/settings/backups/"), ("Restore " ~ (name or "a backup"), "")]) }}{% endblock %} +{% block content %} +{% if not report %} +{{ name or "The backup" }} can't be restored. {{ error }}
+{% else %} +{% include "settings/_backup_report.html" %} + +This Steward is older than the one that made the backup. Update Steward, then restore it.
+ {% elif not report.ok %} +Its key doesn't open all of its secrets, so passwords and keys wouldn't come back. Restore a + different backup.
+ {% else %} +/data/backups and the bucket, so
+ this can be undone the same way.{{ error }}
{% endif %} + + {% endif %} +The restore of {{ restore_failed.name }} + {%- if restore_failed.since %} {{ restore_failed.since|ago }}{% endif %} failed, and nothing changed: + {{ restore_failed.error }}
+{% endif %} + {% if flash %}None yet.
{% endif %}Check compares a backup with what is in place: what
- a restore would bring back, change or remove, area by area. It changes nothing. To bring Steward back
- from one, see steward restore in the backups guide (docs/core/backups.md).
A steward-<time>.tar downloaded from
the bucket, or made with steward backup.
restore to confirm" in page
+ token = _token(page)
+
+ wrong = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+ form={"token": token, "confirm": "yes"})
+ assert wrong.status_code == 200 and not stubs["started"]
+ assert "Type restore to confirm." in await wrong.get_data(as_text=True)
+
+ go = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+ form={"token": token, "confirm": " Restore "})
+ assert go.status_code == 302 and go.headers["Location"].endswith("/settings/backups/")
+ [started] = stubs["started"]
+ assert started["name"] == "steward-20261015T030000Z.tar" and not started["replace_all"]
+ assert started["path"].read_bytes() == b"a backup" and started["owned_dir"].name == token
+
+ gone = await admin.post("/settings/backups/restore/start/", headers=CSRF,
+ form={"token": "0" * 32, "confirm": "restore"})
+ assert "no longer held" in await gone.get_data(as_text=True)
+ asyncio.run(_go())
+
+
+def test_an_older_backup_also_needs_replace_everything(app, stubs):
+ stubs["schema"] = "older"
+
+ async def _go():
+ admin = await _client(app, "admin")
+ page = await (await _upload(admin)).get_data(as_text=True)
+ assert 'name="replace_all"' in page and "replaces everything" in page
+ token = _token(page)
+ await admin.post("/settings/backups/restore/start/", headers=CSRF,
+ form={"token": token, "confirm": "restore"})
+ assert not stubs["started"]
+ await admin.post("/settings/backups/restore/start/", headers=CSRF,
+ form={"token": token, "confirm": "restore", "replace_all": "on"})
+ assert [s["replace_all"] for s in stubs["started"]] == [True]
+ asyncio.run(_go())
+
+
+def test_a_backup_that_cant_go_in_has_no_button(app, stubs):
+ async def _go():
+ admin = await _client(app, "admin")
+ stubs["schema"] = "newer"
+ newer = await (await _upload(admin)).get_data(as_text=True)
+ stubs["schema"], stubs["unopened"] = "same", 1
+ unopened = await (await _upload(admin)).get_data(as_text=True)
+ return newer, unopened
+ newer, unopened = asyncio.run(_go())
+ assert "Update Steward, then restore it." in newer and 'name="token"' not in newer
+ assert "wouldn't come back" in unopened and 'name="token"' not in unopened
+
+
+def test_only_an_admin_restores(app, stubs):
+ async def _go():
+ operator = await _client(app, "operator")
+ return (await _upload(operator)).status_code, (await operator.post(
+ "/settings/backups/restore/", headers=CSRF, form={"name": "steward-20261015T030000Z.tar"})
+ ).status_code
+ assert asyncio.run(_go()) == (403, 403)
+ assert not stubs["started"]
+
+
+def test_the_backups_page_offers_restore_and_says_when_one_failed(app, stubs, monkeypatch):
+ from steward.backup import restoring
+ monkeypatch.setattr(restoring, "STATE", restoring.Restoring(
+ name="steward-20261014T030000Z.tar", error="The restore was undone, nothing changed: x"))
+
+ async def _go():
+ admin = await _client(app, "admin")
+ return await (await admin.get("/settings/backups/")).get_data(as_text=True)
+ page = asyncio.run(_go())
+ assert 'formaction="/settings/backups/restore/file/"' in page
+ assert "The restore of steward-20261014T030000Z.tar" in page and "nothing changed: x" in page