From fb99f8b551286048479682e0baa1153bde3a08fe Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 15 Oct 2026 11:03:01 -0400 Subject: [PATCH] =?UTF-8?q?feat(backup):=20restore=20from=20Settings=20?= =?UTF-8?q?=E2=86=92=20Backups,=20with=20restoring=20mode=20(#5816,=20#581?= =?UTF-8?q?7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5816 "Restoring mode": while a backup goes back in, the scheduler holds, ingest and HTMX answer 503 with Retry-After (agents keep their samples), and every page says what is happening. A safety backup of Steward as it stands is made first, to /data/backups and the bucket. The backup's key is put in place, the restore audited, and Steward exits with code 3 so Swarm, compose or systemd starts it fresh. A failed in-place restore changes nothing and says why on Backups; a full replace that fails after the wipe restarts into first-run setup. #5817 "Settings → Backups: Restore": a Restore button on each sent backup and on the file card. The chosen backup is checked (the same comparison as Check, now a shared partial) and held server-side for an hour under a token; it starts only once the admin types "restore", and an older backup also needs "Replace everything" ticked. A newer backup, or one whose key can't open its secrets, gets no button. Co-Authored-By: Claude Opus 5.5 --- steward/app.py | 7 +- steward/audit/service.py | 2 + steward/auth/csrf.py | 1 + steward/backup/restoring.py | 272 ++++++++++++++++++ steward/cli.py | 5 + steward/core/scheduler.py | 11 +- steward/settings/routes.py | 107 ++++++- steward/templates/errors/restoring.html | 18 ++ .../templates/settings/_backup_report.html | 61 ++++ steward/templates/settings/backup_check.html | 59 +--- .../templates/settings/backup_restore.html | 58 ++++ steward/templates/settings/backups.html | 25 +- tests/core/test_restoring.py | 145 ++++++++++ .../integration/test_backup_restore_routes.py | 159 ++++++++++ 14 files changed, 858 insertions(+), 72 deletions(-) create mode 100644 steward/backup/restoring.py create mode 100644 steward/templates/errors/restoring.html create mode 100644 steward/templates/settings/_backup_report.html create mode 100644 steward/templates/settings/backup_restore.html create mode 100644 tests/core/test_restoring.py create mode 100644 tests/integration/test_backup_restore_routes.py diff --git a/steward/app.py b/steward/app.py index 19dc84e..bd14640 100644 --- a/steward/app.py +++ b/steward/app.py @@ -175,6 +175,10 @@ def create_app( app.jinja_env.globals["csrf_field"] = _csrf.csrf_field # The sign-in page offers "Forgot your password?" only while mail can send (milestone 497). app.jinja_env.globals["reset_by_email"] = lambda: bool(app.config.get("SMTP", {}).get("host")) + # While a backup is being restored nothing else reads or writes the + # database: a page says so, and agents get a 503 to retry (#5816). + from .backup import restoring as _restoring + app.before_request(_restoring.gate) app.before_request(_csrf.check_request) # The session cookie is Secure, and HSTS is sent, only on requests that # arrived over HTTPS — directly or as a trusted proxy reports it @@ -595,5 +599,6 @@ async def _mark_interrupted_runs(app: Quart) -> None: async def _run_scheduler(app: Quart) -> None: + from .backup.restoring import is_restoring from .core.scheduler import start_scheduler - await start_scheduler(app._task_registry) + await start_scheduler(app._task_registry, paused=is_restoring) diff --git a/steward/audit/service.py b/steward/audit/service.py index e7683b8..3e61c8d 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -200,6 +200,8 @@ ACTIONS = { "prometheus.template_deleted": "deleted Prometheus widget template", "backup.made": "sent a backup", "backup.failed": "tried to send a backup, which failed:", + "backup.restored": "restored Steward from a backup", + "backup.restore_failed": "tried to restore Steward from a backup, which failed:", } # The filter's choices: an area, matched as a prefix of the action code. diff --git a/steward/auth/csrf.py b/steward/auth/csrf.py index 561ea1d..d6d8b90 100644 --- a/steward/auth/csrf.py +++ b/steward/auth/csrf.py @@ -52,6 +52,7 @@ def rotate() -> None: # a backup file (settings only is small; an older one carries the audit log). UPLOAD_LIMITS: dict[str, int] = { "/settings/backups/check/file/": 512 * 1024 * 1024, + "/settings/backups/restore/file/": 512 * 1024 * 1024, } diff --git a/steward/backup/restoring.py b/steward/backup/restoring.py new file mode 100644 index 0000000..09efb79 --- /dev/null +++ b/steward/backup/restoring.py @@ -0,0 +1,272 @@ +"""Restoring mode (#5816): what a running Steward does while a backup goes back +in, from Settings → Backups. + +1. Read the backup and refuse it up front if it can't go in: not a backup, + or a different secret key while STEWARD_SECRET_KEY pins this one. +2. Take a safety backup of Steward as it stands: kept on /data/backups and + sent to the bucket when one is set. A restore can then be undone the same + way. Without it nothing goes further. +3. While it runs, Steward is restoring: the scheduler holds its ticks, agent + ingest answers 503 (agents keep their samples and send them again), and + every page says what is happening. +4. In place for a backup of this schema (restore.restore_in_place), or a + full replace the person agreed to for an older one. +5. The backup's key is put in place, the restore audited, and the process + exits with RESTART_EXIT_CODE. Swarm, compose or systemd starts it again, + and settings, the key and plugins load fresh. The person who restored + signs in again: users came back from the backup. + +A restore that fails in place has changed nothing; Steward carries on and +says why. A full replace that fails part-way leaves an empty database, so +Steward restarts into first-run setup, where the safety backup can be +uploaded. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import secrets +import shutil +import signal +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger(__name__) + +SAFETY_DIR = Path("/data/backups") +# Safety backups kept on /data, newest first; older ones are on the bucket. +SAFETY_KEEP = 5 +# Not 0: systemd's Restart=on-failure restarts only a failed exit. +RESTART_EXIT_CODE = 3 +# Long enough for the restoring page's next poll to say it is done. +EXIT_DELAY_SECONDS = 5 +# A backup chosen for restore waits here between its check and the +# confirmation (#5817); one not confirmed within the hour is removed. +STAGED_SECONDS = 3600 + + +@dataclass +class Restoring: + active: bool = False + name: str = "" + by: str = "" + since: datetime | None = None + step: str = "" + error: str = "" + restart_requested: bool = False + + +STATE = Restoring() +_tasks: set[asyncio.Task] = set() + + +def is_restoring() -> bool: + return STATE.active + + +def start_restore(app, path: Path, name: str, *, username: str, replace_all: bool = False, + owned_dir: Path | None = None) -> bool: + """Start restoring `path` in the background; False if a restore is already + under way. `owned_dir`, when given, is removed once it is over.""" + if STATE.active or STATE.restart_requested: + return False + STATE.active, STATE.name, STATE.by = True, name, username + STATE.since, STATE.step, STATE.error = datetime.now(timezone.utc), "Reading the backup", "" + task = asyncio.get_running_loop().create_task( + _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir)) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return True + + +async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, + owned_dir: Path | None) -> None: + from steward.core.audit import log_audit + + from .dump import BackupError + from .s3 import S3Error + wiped = False + try: + key = await asyncio.to_thread(_backup_key, path) + pinned = os.environ.get("STEWARD_SECRET_KEY") + if pinned and pinned != key: + raise BackupError("STEWARD_SECRET_KEY is set and differs from the backup's key; " + "set it to the backup's key first, or its passwords and keys can't be read.") + STATE.step = "Saving a safety backup of Steward as it stands" + safety = await _safety_backup(app) + STATE.step = "Restoring" + plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] + if replace_all: + from .restore import restore + wiped = True + done = await restore(path, app.config["DATABASE_URL"], wipe=True, plugin_dirs=plugin_dirs) + detail = {"mode": "replaced everything"} + else: + from .restore import restore_in_place + done = await restore_in_place(path, app.config["DATABASE_URL"], plugin_dirs=plugin_dirs) + detail = {"mode": "in place", "tables": len(done.tables), "rows": done.rows, + "put_right": done.fixed} + if done.secret_key != app.config["SECRET_KEY"] and not pinned: + _write_key(done.secret_key) + await log_audit(app, None, username, "backup.restored", entity_type="backup", + entity_id=name, detail=detail | {"safety_backup": safety}) + logger.warning("Restored %s (%s); restarting", name, detail["mode"]) + STATE.step = "Restored. Steward is restarting; sign in again when it is back" + _restart_soon() + except (BackupError, S3Error, OSError) as exc: + _failed(app, name, username, str(exc), wiped) + except Exception as exc: + logger.exception("Restore of %s failed", name) + _failed(app, name, username, f"Unexpected error: {exc}", wiped) + finally: + if owned_dir is not None: + shutil.rmtree(owned_dir, ignore_errors=True) + + +def _failed(app, name: str, username: str, error: str, wiped: bool) -> None: + from steward.core.audit import log_audit + logger.error("Restore of %s failed: %s", name, error) + STATE.error = error + if wiped: + # The database was dropped before the restore failed: start again + # into first-run setup, where the safety backup can be uploaded. + STATE.step = ("The restore failed after the database was emptied; Steward is restarting " + "into first-run setup, where the safety backup can be uploaded") + _restart_soon() + return + STATE.active = False + task = asyncio.get_running_loop().create_task(log_audit( + app, None, username, "backup.restore_failed", entity_type="backup", entity_id=name, + detail={"error": error})) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + + +def _backup_key(path: Path) -> str: + import tarfile + + from .dump import KEY_NAME, BackupError + try: + with tarfile.open(path, "r") as tar: + return tar.extractfile(KEY_NAME).read().decode().strip() + except (tarfile.TarError, OSError, KeyError, AttributeError): + raise BackupError(f"{path.name} isn't a Steward backup.") from None + + +async def _safety_backup(app) -> str: + """A backup of Steward as it stands, before anything is replaced: on + /data/backups always, and on the bucket when one is set. Its run is in + Recent backups like any other.""" + from steward.core.settings import get_all_settings + + from .dump import make_backup + from .job import _record, migration_heads, target_from + from .s3 import put_file + SAFETY_DIR.mkdir(parents=True, exist_ok=True) + url = app.config["DATABASE_URL"] + made = await make_backup(url, app.config["SECRET_KEY"], await migration_heads(url), SAFETY_DIR) + async with app.db_sessionmaker() as db: + target = target_from(await get_all_settings(db)) + entry = {"at": datetime.now(timezone.utc).isoformat(), "ok": True, "by": "before restore", + "name": made.name, "size": made.size, "secs": 0.0, "error": ""} + if target is not None: + await put_file(target, made.name, made.path) + else: + entry["error"] = "Kept on /data/backups only: no bucket is set up." + await _record(app, entry, None, "") + _prune() + return made.name + + +def _prune() -> None: + kept = sorted(SAFETY_DIR.glob("steward-*.tar"), reverse=True) + for old in kept[SAFETY_KEEP:]: + old.unlink(missing_ok=True) + + +def _write_key(key: str) -> None: + from steward.config import _SECRET_KEY_FILE + _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True) + _SECRET_KEY_FILE.write_text(key) + _SECRET_KEY_FILE.chmod(0o600) + + +def _restart_soon() -> None: + STATE.restart_requested = True + loop = asyncio.get_running_loop() + loop.call_later(EXIT_DELAY_SECONDS, os.kill, os.getpid(), signal.SIGTERM) + + +# ── Chosen, checked, waiting for a yes (#5817) ───────────────────────────── + +@dataclass +class Staged: + token: str + dir: Path + name: str = "" + schema: str = "" + ok: bool = False + + @property + def path(self) -> Path: + return self.dir / "backup.tar" + + def save(self) -> None: + (self.dir / "staged.json").write_text(json.dumps( + {"name": self.name, "schema": self.schema, "ok": self.ok})) + + +def _staging() -> Path: + return SAFETY_DIR / "incoming" + + +def stage() -> Staged: + """A fresh place to hold one backup until its restore is confirmed. Old + ones not confirmed are removed first.""" + root = _staging() + root.mkdir(parents=True, exist_ok=True) + cutoff = time.time() - STAGED_SECONDS + for old in root.iterdir(): + if old.stat().st_mtime < cutoff: + shutil.rmtree(old, ignore_errors=True) + token = secrets.token_hex(16) + (root / token).mkdir(mode=0o700) + return Staged(token, root / token) + + +def staged(token: str) -> Staged | None: + """The backup held under `token`, or None when there is none (or it was + removed for its age).""" + if not re.fullmatch(r"[0-9a-f]{32}", token or ""): + return None + d = _staging() / token + try: + meta = json.loads((d / "staged.json").read_text()) + except (OSError, ValueError): + return None + if not (d / "backup.tar").is_file(): + return None + return Staged(token, d, meta.get("name", ""), meta.get("schema", ""), bool(meta.get("ok"))) + + +# ── While it runs ─────────────────────────────────────────────────────────── + +async def gate(): + """before_request: while restoring, nothing reads or writes the database. + Agents and scripts get a 503 to retry; a page says what is happening.""" + from quart import Response, g, render_template, request + if not (STATE.active or STATE.restart_requested) or request.path.startswith("/static/"): + return None + g.database_down = True # the page's own reads (the sidebar's alert count) wait + headers = {"Retry-After": "30"} + wants_page = "text/html" in request.headers.get("Accept", "") + if request.method != "GET" or request.headers.get("HX-Request") or not wants_page: + return Response("Steward is restoring a backup; try again shortly.", status=503, + headers=headers, content_type="text/plain; charset=utf-8") + page = await render_template("errors/restoring.html", state=STATE) + return Response(page, status=503, headers=headers) diff --git a/steward/cli.py b/steward/cli.py index bfc098a..6ed361f 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -58,6 +58,11 @@ def main(ctx: click.Context, config: str, host: str, port: int, debug: bool) -> app.run(host=host, port=port, debug=True) return asyncio.run(_serve(app, hypercorn_config(host, port))) + from .backup.restoring import RESTART_EXIT_CODE, STATE as restoring + if restoring.restart_requested: + # A restore finished: exit as failed so every supervisor starts + # Steward again (systemd's Restart=on-failure ignores a clean exit). + raise SystemExit(RESTART_EXIT_CODE) @main.command("backup") diff --git a/steward/core/scheduler.py b/steward/core/scheduler.py index 9379304..5245258 100644 --- a/steward/core/scheduler.py +++ b/steward/core/scheduler.py @@ -60,8 +60,10 @@ class _DueTracker: self.in_flight.discard(name) -async def start_scheduler(tasks: list[ScheduledTask]) -> None: - """Run scheduled tasks in a loop. Call with asyncio.create_task().""" +async def start_scheduler(tasks: list[ScheduledTask], paused=None) -> None: + """Run scheduled tasks in a loop. Call with asyncio.create_task(). + `paused`, when given, is asked each tick: while it says so nothing new + starts (a restore is under way, steward/backup/restoring.py).""" tracker = _DueTracker() def _spawn(task: ScheduledTask, now: float) -> None: @@ -77,8 +79,9 @@ async def start_scheduler(tasks: list[ScheduledTask]) -> None: while True: now = asyncio.get_event_loop().time() - for task in tracker.due(tasks, now): - _spawn(task, now) + if paused is None or not paused(): + for task in tracker.due(tasks, now): + _spawn(task, now) await asyncio.sleep(1) diff --git a/steward/settings/routes.py b/steward/settings/routes.py index c029f4d..af51bfb 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -695,7 +695,7 @@ async def send_report_now(): async def _backups_page(flash=None): """Settings → Backups (#5673): how the newest backup went, the bucket and the daily hour (in the reader's zone), and the runs before.""" - from steward.backup import job + from steward.backup import job, restoring from steward.core.formatting import zone async with current_app.db_sessionmaker() as db: settings = await get_all_settings(db) @@ -705,7 +705,8 @@ async def _backups_page(flash=None): "settings/backups.html", settings=settings, flash=flash, shown_hour=hour, verdict=job.verdict(settings, job.is_running()), history=job.history_rows(settings.get("backup.history")), - has_secret=bool(settings.get("backup.s3_secret_key"))) + has_secret=bool(settings.get("backup.s3_secret_key")), + restore_failed=restoring.STATE if restoring.STATE.error else None) @settings_bp.get("/backups/") @@ -824,6 +825,108 @@ async def check_uploaded_backup(): return await _check_page(report=report, name=name) + +# Restore (#5817): choose a backup, see what it would change (the same check as +# above), then type to confirm. The chosen file is held server-side between +# the two (restoring.stage), so the confirmation names it by a token. + +RESTORE_WORD = "restore" + + +async def _restore_page(report=None, error: str = "", name: str = "", held=None): + return await render_template("settings/backup_restore.html", report=report, error=error, + name=name, held=held, word=RESTORE_WORD) + + +async def _hold_and_check(held, name: str): + from steward.backup import job + from steward.backup.dump import BackupError + try: + report = await job.check_file(current_app._get_current_object(), held.path, name) + except BackupError as exc: + import shutil + shutil.rmtree(held.dir, ignore_errors=True) + return await _restore_page(error=str(exc), name=name) + held.name, held.schema, held.ok = name, report.schema, report.ok + held.save() + return await _restore_page(report=report, name=name, held=held) + + +@settings_bp.post("/backups/restore/") +@require_role(UserRole.admin) +async def restore_stored_backup(): + """Fetch one of the bucket's backups, check it, and ask to confirm.""" + import shutil + + from steward.backup import job, restoring + from steward.backup.dump import BackupError + from steward.backup.s3 import S3Error, get_file + name = (await request.form).get("name", "") + if not job.is_backup_name(name): + return await _restore_page(error=f"{name!r} isn't the name of a Steward backup.", name=name) + async with current_app.db_sessionmaker() as db: + target = job.target_from(await get_all_settings(db)) + if target is None: + return await _restore_page(error="The bucket isn't set up: fill in the endpoint, bucket and keys.", + name=name) + held = restoring.stage() + try: + await get_file(target, name, held.path) + except (BackupError, S3Error) as exc: + shutil.rmtree(held.dir, ignore_errors=True) + error = str(exc) + if "AccessDenied" in error: + error += (" The bucket's key may not read backups back; download the file from the " + "bucket and restore it with Restore a file instead.") + return await _restore_page(error=error, name=name) + return await _hold_and_check(held, name) + + +@settings_bp.post("/backups/restore/file/") +@require_role(UserRole.admin) +async def restore_uploaded_backup(): + """A backup file sent from the browser, checked and held for the + confirmation. Its size limit is raised in auth/csrf.py UPLOAD_LIMITS.""" + from steward.backup import restoring + upload = (await request.files).get("file") + if upload is None or not upload.filename: + return await _restore_page(error="Choose a backup file to restore.") + held = restoring.stage() + await upload.save(held.path) + return await _hold_and_check(held, Path(upload.filename).name) + + +@settings_bp.post("/backups/restore/start/") +@require_role(UserRole.admin) +async def start_restore(): + """The confirmation: the held backup goes back in (restoring.py). Every + page shows the restore until Steward restarts.""" + import shutil + + from steward.backup import job, restoring + form = await request.form + held = restoring.staged(form.get("token", "")) + if held is None: + return await _restore_page(error="That backup is no longer held (it is removed an hour after " + "it was checked). Choose it again.") + if held.schema == "newer" or not held.ok: + shutil.rmtree(held.dir, ignore_errors=True) + return await _restore_page(error="This backup can't be restored here; check it again to see why.", + name=held.name) + replace_all = held.schema == "older" + if form.get("confirm", "").strip().lower() != RESTORE_WORD or ( + replace_all and "replace_all" not in form): + report = await job.check_file(current_app._get_current_object(), held.path, held.name) + return await _restore_page(report=report, name=held.name, held=held, + error=f"Type {RESTORE_WORD} to confirm" + + (", and tick Replace everything." if replace_all else ".")) + started = restoring.start_restore(current_app._get_current_object(), held.path, held.name, + username=session.get("username", "") or "an admin", + replace_all=replace_all, owned_dir=held.dir) + if not started: + return await _restore_page(error="A restore is already under way.", name=held.name) + return redirect(url_for("settings.backups")) + def _build_plugin_cfg_from_form(plugin: dict, form) -> dict: """Extract config values for one plugin from a form submission.""" name = plugin["_dir"] diff --git a/steward/templates/errors/restoring.html b/steward/templates/errors/restoring.html new file mode 100644 index 0000000..12e6b4d --- /dev/null +++ b/steward/templates/errors/restoring.html @@ -0,0 +1,18 @@ +{# errors/restoring.html — every page while a backup is being restored (#5816): + what is going back in, by whom, the step it is on. It reloads itself, so + the sign-in page appears once Steward is back. #} +{% extends "base.html" %} +{% block title %}Restoring a backup — Steward{% endblock %} +{% block head %}{% endblock %} +{% block content %} +
+

Steward is restoring a backup

+

{{ state.name }}{% if state.by %}, started by {{ state.by }}{% endif %} + {%- if state.since %} {{ state.since|ago }}{% endif %}.

+

{{ state.step }}…

+

+ Nothing else reads or writes Steward's records until it is done. Agents keep their readings + and send them when Steward is back. This page reloads by itself. +

+
+{% endblock %} diff --git a/steward/templates/settings/_backup_report.html b/steward/templates/settings/_backup_report.html new file mode 100644 index 0000000..395456d --- /dev/null +++ b/steward/templates/settings/_backup_report.html @@ -0,0 +1,61 @@ +{# settings/_backup_report.html — what a check found (#5823), shown on the check + page and on the restore confirm page (#5817): the verdict, what the backup + is, each area's count, then each difference. Needs `report`. #} +{% set tone = "tone-crit" if not report.ok else ("tone-ok" if not report.changes else "tone-warn") %} +

{{ report.summary() }}

+ +
+
+
File
{{ report.name }}
+
Made
{% if report.made_at %}{{ report.made_at|when }}{% else %}—{% endif %} + by Steward {{ report.manifest.steward_version or "—" }}
+
Schema
+ {%- if report.schema == "same" %}The same as this Steward's. + {%- elif report.schema == "older" %}Older than this Steward's. A restore would replace everything, and Steward would bring it up to date on start. + {%- else %}Newer than this Steward's. This Steward can't restore it; update Steward first.{% endif %}
+
Secret key
+ {%- if report.key_same %}The one in use here.{% else %}A different one. A restore would put it in place, and passwords and keys stored since would need entering again.{% endif %} + {%- if report.sealed %} It opens {{ report.sealed - report.unopened }} of the backup's {{ report.sealed }} sealed value{{ "s" if report.sealed != 1 }}.{% endif %}
+
+
+ +
+ + + + {% for a in report.areas %} + {% set differ = a.tables|sum(attribute="differ", start=[])|length %} + {% set only_b = a.tables|sum(attribute="only_backup", start=[])|length %} + {% set only_h = a.tables|sum(attribute="only_here", start=[])|length %} + + + + + + + + {% endfor %} + +
AreaSameDifferOnly in the backupOnly here
{{ a.name }}{{ a.same }}{% if differ %}{{ differ }}{% else %}—{% endif %}{% if only_b %}{{ only_b }}{% else %}—{% endif %}{% if only_h %}{{ only_h }}{% else %}—{% endif %}
+
+ +{% for a in report.areas if a.changes or a.tables|selectattr("note")|list %} +

{{ a.name }}

+ +{% endfor %} + +

Left out of the comparison: what Steward records as it runs + (when an agent or monitor was last seen, a schedule's last run, the backup history), since it changes by the + minute. A restore still brings those back as they were in the backup.

diff --git a/steward/templates/settings/backup_check.html b/steward/templates/settings/backup_check.html index f4ef323..64be378 100644 --- a/steward/templates/settings/backup_check.html +++ b/steward/templates/settings/backup_check.html @@ -11,64 +11,7 @@ {% if error %}

{{ name or "The backup" }} couldn't be checked. {{ error }}

{% else %} -{% set tone = "tone-crit" if not report.ok else ("tone-ok" if not report.changes else "tone-warn") %} -

{{ report.summary() }}

- -
-
-
File
{{ report.name }}
-
Made
{% if report.made_at %}{{ report.made_at|when }}{% else %}—{% endif %} - by Steward {{ report.manifest.steward_version or "—" }}
-
Schema
- {%- if report.schema == "same" %}The same as this Steward's. - {%- elif report.schema == "older" %}Older than this Steward's. A restore would replace everything, and Steward would bring it up to date on start. - {%- else %}Newer than this Steward's. This Steward can't restore it; update Steward first.{% endif %}
-
Secret key
- {%- if report.key_same %}The one in use here.{% else %}A different one. A restore would put it in place, and passwords and keys stored since would need entering again.{% endif %} - {%- if report.sealed %} It opens {{ report.sealed - report.unopened }} of the backup's {{ report.sealed }} sealed value{{ "s" if report.sealed != 1 }}.{% endif %}
-
-
- -
- - - - {% for a in report.areas %} - {% set differ = a.tables|sum(attribute="differ", start=[])|length %} - {% set only_b = a.tables|sum(attribute="only_backup", start=[])|length %} - {% set only_h = a.tables|sum(attribute="only_here", start=[])|length %} - - - - - - - - {% endfor %} - -
AreaSameDifferOnly in the backupOnly here
{{ a.name }}{{ a.same }}{% if differ %}{{ differ }}{% else %}—{% endif %}{% if only_b %}{{ only_b }}{% else %}—{% endif %}{% if only_h %}{{ only_h }}{% else %}—{% endif %}
-
- -{% for a in report.areas if a.changes or a.tables|selectattr("note")|list %} -

{{ a.name }}

- -{% endfor %} - -

Left out of the comparison: what Steward records as it runs - (when an agent or monitor was last seen, a schedule's last run, the backup history), since it changes by the - minute. A restore still brings those back as they were in the backup.

+{% include "settings/_backup_report.html" %} {% endif %}

Back to Backups

{% endblock %} diff --git a/steward/templates/settings/backup_restore.html b/steward/templates/settings/backup_restore.html new file mode 100644 index 0000000..69f27ee --- /dev/null +++ b/steward/templates/settings/backup_restore.html @@ -0,0 +1,58 @@ +{# settings/backup_restore.html — restoring a backup, before anything changes + (#5817): what it would bring back (the same check as Check, #5823), what a + restore does, then a typed confirmation. A backup that can't go in says + why and offers no button. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% from "settings/_tabs.html" import settings_tabs %} +{% block title %}Settings — Backups — Restore — Steward{% endblock %} +{% block section_tabs %}{{ settings_tabs("backups") }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Backups", "/settings/backups/"), ("Restore " ~ (name or "a backup"), "")]) }}{% endblock %} +{% block content %} +{% if not report %} +

{{ name or "The backup" }} can't be restored. {{ error }}

+{% else %} +{% include "settings/_backup_report.html" %} + +
+

Restore {{ name }}

+ {% if report.schema == "newer" %} +

This Steward is older than the one that made the backup. Update Steward, then restore it.

+ {% elif not report.ok %} +

Its key doesn't open all of its secrets, so passwords and keys wouldn't come back. Restore a + different backup.

+ {% else %} +
    +
  • A safety backup of Steward as it stands is made first, on /data/backups and the bucket, so + this can be undone the same way.
    • + {% if report.schema == "same" %} +
  • Settings, users, hosts, monitors, rules and dashboards become the backup's. The audit log and the history + of what is still here stay; history of anything the backup doesn't have goes with it.
    • +
  • If anything goes wrong part-way, nothing changes.
    • + {% else %} +
  • The backup is from an older Steward, so it replaces everything: the audit + log, alert history, metrics and check results go too, and Steward brings the backup up to date as it + starts. If this fails part-way, Steward starts empty, at first-run setup, where the safety backup can be + uploaded.
    • + {% endif %} +
  • While it runs, pages say Steward is restoring and agents hold their readings. Steward then restarts, and + you sign in again as a user from the backup.
    • +
+ {% if error %}

{{ error }}

{% endif %} +
{{ csrf_field() }} + + {% if report.schema == "older" %} + + {% endif %} +
+ + +
+ +
+ {% endif %} +
+{% endif %} +

Back to Backups

+{% endblock %} diff --git a/steward/templates/settings/backups.html b/steward/templates/settings/backups.html index bdb9fe1..1612cde 100644 --- a/steward/templates/settings/backups.html +++ b/steward/templates/settings/backups.html @@ -1,7 +1,7 @@ {# settings/backups.html — Steward's own backups (#5673): a verdict, the bucket they go to (Fabled Archives over S3) and the daily hour, then the recent - runs, each with its own check and a Check button (#5823), and a file to - check. Restoring is `steward restore`; see docs/core/backups.md. #} + runs, each with its own check and Check and Restore buttons (#5823, #5817), + and a file to check or restore. #} {% extends "base.html" %} {% from "_macros.html" import crumbs %} {% from "settings/_tabs.html" import settings_tabs %} @@ -16,6 +16,12 @@ {% if verdict.good %}The newest good backup was sent {{ verdict.good|ago }}.{% else %}No backup has been sent yet.{% endif %} {%- endif %}

+{% if restore_failed %} +

The restore of {{ restore_failed.name }} + {%- if restore_failed.since %} {{ restore_failed.since|ago }}{% endif %} failed, and nothing changed: + {{ restore_failed.error }}

+{% endif %} + {% if flash %}
{{ r.size|bytes if r.ok else "—" }} {{ r.secs|duration }} {{ "the schedule" if r.by == "schedule" else r.by }} - {% if r.ok and r.name %} + {% if r.ok and r.name %}
{{ csrf_field() }} -
{% endif %} + +
{{ csrf_field() }} + + +
{% endif %} {% endfor %} @@ -144,15 +154,16 @@

None yet.

{% endif %}

Check compares a backup with what is in place: what - a restore would bring back, change or remove, area by area. It changes nothing. To bring Steward back - from one, see steward restore in the backups guide (docs/core/backups.md).

+ a restore would bring back, change or remove, area by area. It changes nothing. Restore shows the same + comparison and asks you to confirm before anything changes.

-

Check a file

+

Check or restore a file

{{ csrf_field() }} +

A steward-<time>.tar downloaded from the bucket, or made with steward backup.

diff --git a/tests/core/test_restoring.py b/tests/core/test_restoring.py new file mode 100644 index 0000000..862cf90 --- /dev/null +++ b/tests/core/test_restoring.py @@ -0,0 +1,145 @@ +"""Restoring mode (steward/backup/restoring.py, #5816): while a backup goes back +in, pages say so and agents are told to retry; the scheduler holds; a restore +saves a safety backup first, puts the backup's key in place, audits itself +and asks for a restart; a failed one says why and changes nothing.""" +import asyncio +from types import SimpleNamespace + +import pytest + +from steward.backup import restoring +from steward.backup.dump import BackupError + + +@pytest.fixture(autouse=True) +def fresh_state(monkeypatch): + monkeypatch.setattr(restoring, "STATE", restoring.Restoring()) + + +async def test_while_restoring_pages_say_so_and_agents_are_told_to_retry(client): + restoring.STATE.active, restoring.STATE.name = True, "steward-20261015T030000Z.tar" + restoring.STATE.by, restoring.STATE.step = "admin", "Restoring" + page = await client.get("/", headers={"Accept": "text/html"}) + assert page.status_code == 503 and page.headers["Retry-After"] == "30" + body = await page.get_data(as_text=True) + assert "Steward is restoring a backup" in body and "steward-20261015T030000Z.tar" in body + assert 'http-equiv="refresh"' in body + + ingest = await client.post("/plugins/host_agent/ingest", json={}) + assert ingest.status_code == 503 and ingest.headers["Retry-After"] == "30" + assert "restoring a backup" in await ingest.get_data(as_text=True) + + +async def test_when_not_restoring_the_gate_stays_out_of_the_way(app): + async with app.test_request_context("/", headers={"Accept": "text/html"}): + assert await restoring.gate() is None + + +async def test_the_scheduler_holds_while_paused(): + from steward.core.scheduler import ScheduledTask, start_scheduler + ran = [] + paused = {"now": True} + + async def tick(): + ran.append(1) + + task = asyncio.create_task(start_scheduler( + [ScheduledTask("t", tick, interval_seconds=0, run_on_startup=False)], + paused=lambda: paused["now"])) + await asyncio.sleep(1.5) + held = len(ran) + paused["now"] = False + await asyncio.sleep(1.5) + task.cancel() + assert held == 0 and ran + + +def _app(): + return SimpleNamespace(config={"DATABASE_URL": "postgresql+asyncpg://x/y", "SECRET_KEY": "k-here", + "PLUGIN_DIRS": ["plugins"]}) + + +def _stub(monkeypatch, tmp_path, *, restore_error=None, key="k-backup"): + from steward import config + from steward.backup import restore + from steward.core import audit + seen = {"audits": [], "restarts": 0} + + async def safety(app): + seen["safety"] = True + return "steward-20261015T140000Z.tar" + + async def in_place(path, url, plugin_dirs=None): + if restore_error: + raise BackupError(restore_error) + return restore.RestoredInPlace({}, key, ["app_settings", "hosts"], 12, 1) + + async def log_audit(app, user_id, username, action, **kw): + seen["audits"].append((action, kw.get("detail"))) + + monkeypatch.setattr(restoring, "_backup_key", lambda path: key) + monkeypatch.setattr(restoring, "_safety_backup", safety) + monkeypatch.setattr(restore, "restore_in_place", in_place) + monkeypatch.setattr(audit, "log_audit", log_audit) + monkeypatch.setattr(restoring, "_restart_soon", lambda: seen.__setitem__("restarts", seen["restarts"] + 1)) + monkeypatch.setattr(config, "_SECRET_KEY_FILE", tmp_path / "secret.key") + monkeypatch.delenv("STEWARD_SECRET_KEY", raising=False) + return seen + + +async def _restore(tmp_path): + assert restoring.start_restore(_app(), tmp_path / "b.tar", "steward-20261015T030000Z.tar", + username="admin") + await asyncio.gather(*list(restoring._tasks)) + await asyncio.gather(*list(restoring._tasks)) # the failure audit, when there is one + + +async def test_a_restore_saves_a_safety_backup_puts_the_key_in_place_and_restarts(monkeypatch, tmp_path): + seen = _stub(monkeypatch, tmp_path) + await _restore(tmp_path) + assert seen["safety"] and seen["restarts"] == 1 + assert (tmp_path / "secret.key").read_text() == "k-backup" + [(action, detail)] = seen["audits"] + assert action == "backup.restored" + assert detail == {"mode": "in place", "tables": 2, "rows": 12, "put_right": 1, + "safety_backup": "steward-20261015T140000Z.tar"} + assert restoring.STATE.step.startswith("Restored") + + +async def test_a_failed_restore_says_why_and_steward_carries_on(monkeypatch, tmp_path): + seen = _stub(monkeypatch, tmp_path, restore_error="The restore was undone, nothing changed: x") + await _restore(tmp_path) + assert not restoring.STATE.active and "undone" in restoring.STATE.error + assert seen["restarts"] == 0 and not (tmp_path / "secret.key").exists() + assert [a for a, _ in seen["audits"]] == ["backup.restore_failed"] + + +async def test_a_pinned_key_that_differs_stops_it_before_anything_is_saved(monkeypatch, tmp_path): + seen = _stub(monkeypatch, tmp_path) + monkeypatch.setenv("STEWARD_SECRET_KEY", "k-pinned") + await _restore(tmp_path) + assert "STEWARD_SECRET_KEY" in restoring.STATE.error and "safety" not in seen + + +async def test_one_restore_at_a_time(monkeypatch, tmp_path): + _stub(monkeypatch, tmp_path) + restoring.STATE.active = True + assert not restoring.start_restore(_app(), tmp_path / "b.tar", "x", username="admin") + + +def test_a_chosen_backup_is_held_by_token_until_it_is_too_old(monkeypatch, tmp_path): + import os + import time + monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path) + held = restoring.stage() + held.path.write_bytes(b"x") + held.name, held.schema, held.ok = "steward-20261015T030000Z.tar", "same", True + held.save() + again = restoring.staged(held.token) + assert (again.name, again.schema, again.ok, again.path) == (held.name, "same", True, held.path) + assert restoring.staged("../../etc") is None and restoring.staged("0" * 32) is None + + old = time.time() - restoring.STAGED_SECONDS - 1 + os.utime(held.dir, (old, old)) + restoring.stage() + assert restoring.staged(held.token) is None diff --git a/tests/integration/test_backup_restore_routes.py b/tests/integration/test_backup_restore_routes.py new file mode 100644 index 0000000..32fe51e --- /dev/null +++ b/tests/integration/test_backup_restore_routes.py @@ -0,0 +1,159 @@ +"""Integration: Restore on Settings → Backups (#5817). A chosen backup is +checked and held; nothing starts until the person types the word, an older +backup also needs Replace everything ticked, a newer one has no button, and +only an admin gets this far. The restore itself is restoring.py's (stubbed +here; tests/core/test_restoring.py covers it).""" +from __future__ import annotations + +import asyncio +import io +import os +import time +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +CSRF = {"X-CSRF-Token": "test-token"} + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +async def _client(app, role): + from steward.models.users import User, UserRole + uid, tag = str(uuid.uuid4()), uuid.uuid4().hex[:8] + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(User(id=uid, username=f"rs-{tag}", email=f"rs-{tag}@test", + password_hash="x", role=UserRole(role), is_active=True)) + client = app.test_client() + async with client.session_transaction() as sess: + sess.update(user_id=uid, user_role=role, username=f"rs-{tag}", + _csrf="test-token", login_at=time.time()) + return client + + +def _report(schema="same", unopened=0): + from steward.backup.check import AreaCheck, CheckReport, TableCheck + return CheckReport(name="steward-20261015T030000Z.tar", + manifest={"created_at": "2026-10-15T03:00:00+00:00"}, schema=schema, + backup_heads=["a"], here_heads=["a"], key_same=True, sealed=1, + unopened=unopened, + areas=[AreaCheck("Settings", [TableCheck("app_settings", same=4)])]) + + +@pytest.fixture +def stubs(monkeypatch, tmp_path): + from steward.backup import job, restoring + seen = {"started": [], "schema": "same", "unopened": 0} + + async def check_file(app, path, name=""): + seen["checked"] = path.read_bytes() + return _report(seen["schema"], seen["unopened"]) + + def start_restore(app, path, name, *, username, replace_all=False, owned_dir=None): + seen["started"].append({"path": path, "name": name, "replace_all": replace_all, + "owned_dir": owned_dir}) + return True + + monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path) + monkeypatch.setattr(job, "check_file", check_file) + monkeypatch.setattr(restoring, "start_restore", start_restore) + return seen + + +async def _upload(client, name="steward-20261015T030000Z.tar"): + from quart.datastructures import FileStorage + return await client.post("/settings/backups/restore/file/", headers=CSRF, files={ + "file": FileStorage(io.BytesIO(b"a backup"), filename=name)}) + + +def _token(page: str) -> str: + import re + return re.search(r'name="token" value="([0-9a-f]{32})"', page).group(1) + + +def test_nothing_starts_until_the_word_is_typed(app, stubs): + async def _go(): + admin = await _client(app, "admin") + page = await (await _upload(admin)).get_data(as_text=True) + assert stubs["checked"] == b"a backup" + assert "Matches what is in place" in page and "Type restore to confirm" in page + token = _token(page) + + wrong = await admin.post("/settings/backups/restore/start/", headers=CSRF, + form={"token": token, "confirm": "yes"}) + assert wrong.status_code == 200 and not stubs["started"] + assert "Type restore to confirm." in await wrong.get_data(as_text=True) + + go = await admin.post("/settings/backups/restore/start/", headers=CSRF, + form={"token": token, "confirm": " Restore "}) + assert go.status_code == 302 and go.headers["Location"].endswith("/settings/backups/") + [started] = stubs["started"] + assert started["name"] == "steward-20261015T030000Z.tar" and not started["replace_all"] + assert started["path"].read_bytes() == b"a backup" and started["owned_dir"].name == token + + gone = await admin.post("/settings/backups/restore/start/", headers=CSRF, + form={"token": "0" * 32, "confirm": "restore"}) + assert "no longer held" in await gone.get_data(as_text=True) + asyncio.run(_go()) + + +def test_an_older_backup_also_needs_replace_everything(app, stubs): + stubs["schema"] = "older" + + async def _go(): + admin = await _client(app, "admin") + page = await (await _upload(admin)).get_data(as_text=True) + assert 'name="replace_all"' in page and "replaces everything" in page + token = _token(page) + await admin.post("/settings/backups/restore/start/", headers=CSRF, + form={"token": token, "confirm": "restore"}) + assert not stubs["started"] + await admin.post("/settings/backups/restore/start/", headers=CSRF, + form={"token": token, "confirm": "restore", "replace_all": "on"}) + assert [s["replace_all"] for s in stubs["started"]] == [True] + asyncio.run(_go()) + + +def test_a_backup_that_cant_go_in_has_no_button(app, stubs): + async def _go(): + admin = await _client(app, "admin") + stubs["schema"] = "newer" + newer = await (await _upload(admin)).get_data(as_text=True) + stubs["schema"], stubs["unopened"] = "same", 1 + unopened = await (await _upload(admin)).get_data(as_text=True) + return newer, unopened + newer, unopened = asyncio.run(_go()) + assert "Update Steward, then restore it." in newer and 'name="token"' not in newer + assert "wouldn't come back" in unopened and 'name="token"' not in unopened + + +def test_only_an_admin_restores(app, stubs): + async def _go(): + operator = await _client(app, "operator") + return (await _upload(operator)).status_code, (await operator.post( + "/settings/backups/restore/", headers=CSRF, form={"name": "steward-20261015T030000Z.tar"}) + ).status_code + assert asyncio.run(_go()) == (403, 403) + assert not stubs["started"] + + +def test_the_backups_page_offers_restore_and_says_when_one_failed(app, stubs, monkeypatch): + from steward.backup import restoring + monkeypatch.setattr(restoring, "STATE", restoring.Restoring( + name="steward-20261014T030000Z.tar", error="The restore was undone, nothing changed: x")) + + async def _go(): + admin = await _client(app, "admin") + return await (await admin.get("/settings/backups/")).get_data(as_text=True) + page = asyncio.run(_go()) + assert 'formaction="/settings/backups/restore/file/"' in page + assert "The restore of steward-20261014T030000Z.tar" in page and "nothing changed: x" in page