From e445c83e676392829bcda1cc2d58ebf5e777b85e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 15 Oct 2026 11:05:39 -0400 Subject: [PATCH] feat(backup): restore at first-run setup, restore --in-place, docs lead with the UI (#5818, #5819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5818 "First-run setup: Restore from a backup, by upload": beside creating the first admin, setup offers "Or restore from a backup". Only while no user exists; the upload is checked (a newer backup or one whose key can't open its secrets is refused and not kept), then restored with no safety backup (nothing to keep), and Steward restarts into the backup's users. #5819 "Docs and CI": backups.md leads with restoring from Settings → Backups and from setup; `steward restore --in-place` does the same-schema restore from the shell, and CI's image check now uses it with a real pg_restore: a changed setting comes back and the audit log stays. Also exempts errors/restoring.html from the breadcrumb guard, as with the other whole-app error pages: it replaces every page while a restore runs, so there is nowhere to trail back to. Co-Authored-By: Claude Opus 5.5 --- .forgejo/workflows/ci.yml | 6 +++ docs/core/backups.md | 62 +++++++++++++++++++++++++--- steward/auth/csrf.py | 1 + steward/auth/routes.py | 44 ++++++++++++++++++++ steward/backup/restoring.py | 18 ++++---- steward/cli.py | 28 +++++++++---- steward/templates/auth/setup.html | 16 ++++++++ tests/test_auth.py | 68 +++++++++++++++++++++++++++++++ tests/test_breadcrumbs.py | 1 + 9 files changed, 225 insertions(+), 19 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 3a70543..1e0ab00 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -240,6 +240,12 @@ jobs: # (last-seen times, backup history) left out of the comparison. docker exec -u app "$SMOKE-app" sh -c 'steward check-backup /tmp/backups/steward-*.tar' | tee /tmp/smoke-check.txt grep -q "Matches what is in place" /tmp/smoke-check.txt + # Put back in place over a changed setting (#5815): the setting comes + # back, and history the backup leaves out (the audit log) stays. + sql -d steward -c "UPDATE app_settings SET value_json = '\"changed\"' WHERE key = 'smoke.backup'" + docker exec -u app "$SMOKE-app" sh -c 'steward restore --in-place /tmp/backups/steward-*.tar' + test "$(sql -d steward -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"' + test "$(sql -d steward -c "SELECT count(*) FROM audit_events WHERE id = 'smoke-audit'")" = "1" sql -d steward -c "CREATE DATABASE restored" docker exec -u app \ -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \ diff --git a/docs/core/backups.md b/docs/core/backups.md index 3a34c05..3822795 100644 --- a/docs/core/backups.md +++ b/docs/core/backups.md @@ -1,8 +1,9 @@ # Backups: Steward backing itself up, and coming back Steward sends a backup of itself to an S3 bucket once a day. Fabled Archives -is the intended target. A backup is one file, and `steward restore` brings -Steward back from it into an empty database. +is the intended target. A backup is one file. Settings → Backups restores it +on a running Steward, first-run setup restores it on a new one, and +`steward restore` brings Steward back from it when Steward can't start. ## What a backup holds @@ -126,8 +127,57 @@ as Steward itself. ## Restoring +### From the web UI + +**On a running Steward: Settings → Backups.** Press **Restore** on one of +the recent backups, or choose a file on the "Check or restore a file" card. +Steward first shows what the backup would change: the same comparison as +Check. Then it says what a restore does and asks you to type `restore`. + +- **A backup of this Steward's schema goes back in place.** Settings, + users, hosts, monitors, alert rules and dashboards become the backup's. + The audit log and the history of everything still here stay. History of + anything the backup doesn't have goes with it. It is one transaction: if + anything fails part-way, nothing changes, and Backups says why. +- **A backup from an older Steward replaces everything**, history included, + and you tick "Replace everything" as well. Steward brings it up to date + as it starts. +- **A backup from a newer Steward isn't offered.** Update Steward first. + +Before anything changes, Steward makes a **safety backup** of itself as it +stands. It goes to `/data/backups` (the newest five are kept) and to the +bucket, and it is listed under Recent backups as "before restore". A +restore can be undone by restoring that. + +While the restore runs: +- every page says Steward is restoring; +- agents get a 503 and keep their readings until Steward is back; +- scheduled work holds. + +Then Steward restarts (it exits with code 3 so Swarm, compose or systemd +starts it again), and you sign in as a user from the backup. If an older +backup's full replace fails after the database was emptied, Steward starts +at first-run setup, where the safety backup can be uploaded. + +**On a new Steward: first-run setup.** Next to "Create the initial admin +account" is **Or restore from a backup**: upload a backup file instead, +and Steward comes up with its users, settings and key. There is nothing to +keep yet, so no safety backup is made. + +The backup's secret key is put in place as `/data/secret.key`. If +`STEWARD_SECRET_KEY` is set instead, it must equal the backup's key, or the +restore stops before anything changes. + +### From the command line + `steward restore` writes into an **empty** database. Pass `--wipe` to drop -everything in the database's public schema first. +everything in the database's public schema first. Use it when Steward +can't start at all. + +`steward restore --in-place` does what Settings → Backups does for a backup +of the same schema, from the shell: it replaces what the backup carries and +keeps the audit log and history, in one transaction. Restart Steward after +it so the restored settings load. 1. Stop Steward (scale its service to 0) so nothing writes during the restore. 2. Download the backup file from Archives. @@ -159,5 +209,7 @@ Postgres 16 with one setting line dropped (`transaction_timeout`, which 16 doesn't know). CI checks the whole path on every build. The built image backs up its own -database, restores into a second one, checks a setting came back and the -metrics table is empty, then starts Steward on it. +database and checks the backup against it. It changes a setting and puts +the backup back in place: the setting returns and the audit log stays. +Then it restores into a second, empty database, checks a setting came back +and the metrics table and audit log are empty, and starts Steward on it. diff --git a/steward/auth/csrf.py b/steward/auth/csrf.py index d6d8b90..d6b19d4 100644 --- a/steward/auth/csrf.py +++ b/steward/auth/csrf.py @@ -53,6 +53,7 @@ def rotate() -> None: UPLOAD_LIMITS: dict[str, int] = { "/settings/backups/check/file/": 512 * 1024 * 1024, "/settings/backups/restore/file/": 512 * 1024 * 1024, + "/setup/restore": 512 * 1024 * 1024, } diff --git a/steward/auth/routes.py b/steward/auth/routes.py index 066f692..abbaa78 100644 --- a/steward/auth/routes.py +++ b/steward/auth/routes.py @@ -295,6 +295,50 @@ async def setup_post(): return redirect(url_for("dashboard.index")) + +@auth_bp.post("/setup/restore") +@public_route("first-run setup's restore (refuses once a user exists)") +async def setup_restore(): + """Bring a new Steward up from a backup instead of an empty start (#5818): + only while no user exists, so there is nothing here to lose and no + safety backup to take. The upload's size limit is raised in + auth/csrf.py UPLOAD_LIMITS. The restore then runs as from Settings → + Backups (backup/restoring.py) and Steward restarts into the backup's + users.""" + import shutil + from pathlib import Path + + from quart import current_app + from steward.backup import job, restoring + from steward.backup.dump import BackupError + if await get_user_count(current_app) > 0: + return redirect(url_for("auth.login")) + + async def refused(error: str): + return await render_template("auth/setup.html", default_url=request.host_url.rstrip("/"), + restore_error=error), 400 + + upload = (await request.files).get("file") + if upload is None or not upload.filename: + return await refused("Choose a backup file to restore.") + name = Path(upload.filename).name + held = restoring.stage() + await upload.save(held.path) + try: + report = await job.check_file(current_app._get_current_object(), held.path, name) + except BackupError as exc: + shutil.rmtree(held.dir, ignore_errors=True) + return await refused(str(exc)) + if report.schema == "newer" or not report.ok: + shutil.rmtree(held.dir, ignore_errors=True) + return await refused("This Steward is older than the one that made the backup; update Steward, " + "then restore it." if report.schema == "newer" else report.summary()) + if not restoring.start_restore(current_app._get_current_object(), held.path, name, + username="first-run setup", replace_all=report.schema == "older", + owned_dir=held.dir, safety=False): + return await refused("A restore is already under way.") + return redirect(url_for("auth.setup")) + # ── Invites (milestone 496) ────────────────────────────────────────────────── _INVITE_GONE = ("This invite link has expired, been used or been withdrawn. " diff --git a/steward/backup/restoring.py b/steward/backup/restoring.py index 09efb79..aad27a0 100644 --- a/steward/backup/restoring.py +++ b/steward/backup/restoring.py @@ -70,22 +70,24 @@ def is_restoring() -> bool: def start_restore(app, path: Path, name: str, *, username: str, replace_all: bool = False, - owned_dir: Path | None = None) -> bool: + owned_dir: Path | None = None, safety: bool = True) -> bool: """Start restoring `path` in the background; False if a restore is already - under way. `owned_dir`, when given, is removed once it is over.""" + under way. `owned_dir`, when given, is removed once it is over. `safety` + False skips the safety backup: first-run setup has nothing to keep.""" if STATE.active or STATE.restart_requested: return False STATE.active, STATE.name, STATE.by = True, name, username STATE.since, STATE.step, STATE.error = datetime.now(timezone.utc), "Reading the backup", "" task = asyncio.get_running_loop().create_task( - _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir)) + _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir, + safety=safety)) _tasks.add(task) task.add_done_callback(_tasks.discard) return True async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, - owned_dir: Path | None) -> None: + owned_dir: Path | None, safety: bool = True) -> None: from steward.core.audit import log_audit from .dump import BackupError @@ -97,8 +99,10 @@ async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, if pinned and pinned != key: raise BackupError("STEWARD_SECRET_KEY is set and differs from the backup's key; " "set it to the backup's key first, or its passwords and keys can't be read.") - STATE.step = "Saving a safety backup of Steward as it stands" - safety = await _safety_backup(app) + safety_name = "" + if safety: + STATE.step = "Saving a safety backup of Steward as it stands" + safety_name = await _safety_backup(app) STATE.step = "Restoring" plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] if replace_all: @@ -114,7 +118,7 @@ async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, if done.secret_key != app.config["SECRET_KEY"] and not pinned: _write_key(done.secret_key) await log_audit(app, None, username, "backup.restored", entity_type="backup", - entity_id=name, detail=detail | {"safety_backup": safety}) + entity_id=name, detail=detail | ({"safety_backup": safety_name} if safety else {})) logger.warning("Restored %s (%s); restarting", name, detail["mode"]) STATE.step = "Restored. Steward is restarting; sign in again when it is back" _restart_soon() diff --git a/steward/cli.py b/steward/cli.py index 6ed361f..63f817d 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -125,24 +125,37 @@ def check_backup_command(ctx: click.Context, backup_file: Path) -> None: @click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.option("--wipe", is_flag=True, default=False, help="Drop everything in the database first. Without it the database must be empty.") +@click.option("--in-place", "in_place", is_flag=True, default=False, + help="Replace what the backup carries in a database of the same schema, keeping " + "the audit log and history (what Settings → Backups does).") @click.pass_context -def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None: +def restore_command(ctx: click.Context, backup_file: Path, wipe: bool, in_place: bool) -> None: """Bring Steward back from a backup file into its (empty) database, and put the backup's secret key in place. Start Steward afterwards: it runs - any migrations the backup is behind on.""" + any migrations the backup is behind on. With --in-place, the database + keeps its history and only what the backup carries is replaced.""" from .backup.dump import BackupError - from .backup.restore import restore + from .backup.restore import restore, restore_in_place from .config import _SECRET_KEY_FILE, _env, load_bootstrap + if wipe and in_place: + raise click.UsageError("--wipe and --in-place don't go together.") boot = load_bootstrap(Path(ctx.obj["config"])) + plugin_dirs = [Path(d) for d in boot["plugin_dirs"]] try: - done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe, - plugin_dirs=[Path(d) for d in boot["plugin_dirs"]])) + if in_place: + done = asyncio.run(restore_in_place(backup_file, boot["database_url"], plugin_dirs=plugin_dirs)) + else: + done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe, plugin_dirs=plugin_dirs)) except BackupError as exc: raise click.ClickException(str(exc)) from None made = done.manifest.get("created_at", "?") click.echo(f"Restored the backup made {made} (Steward {done.manifest.get('steward_version', '?')}).") - if done.fixed: + if in_place: + click.echo(f"Replaced {len(done.tables)} tables ({done.rows} rows); the audit log and history stayed.") + if done.fixed: + click.echo(f"Put right {done.fixed} rows that pointed at something the backup doesn't have.") + elif done.fixed: click.echo(f"Cleared {done.fixed} references to rows a backup leaves out (e.g. Ansible runs).") env_key = _env("SECRET_KEY") if env_key: @@ -155,4 +168,5 @@ def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None: _SECRET_KEY_FILE.write_text(done.secret_key) _SECRET_KEY_FILE.chmod(0o600) click.echo(f"Wrote the backup's secret key to {_SECRET_KEY_FILE}.") - click.echo("Start Steward now; it brings the database up to date on start.") + click.echo("Restart Steward now so it loads them." if in_place + else "Start Steward now; it brings the database up to date on start.") diff --git a/steward/templates/auth/setup.html b/steward/templates/auth/setup.html index d5e4929..9e8cf6b 100644 --- a/steward/templates/auth/setup.html +++ b/steward/templates/auth/setup.html @@ -1,3 +1,5 @@ +{# auth/setup.html — first-run setup: create the first admin, or restore a + backup instead (#5818). Only shown while no user exists. #} {% extends "base.html" %} {% block title %}First-Run Setup — Steward{% endblock %} {% block content %} @@ -32,5 +34,19 @@ + + {# Or come back from a backup (#5818): its users, settings and key. #} +
+

Or restore from a backup

+

Moving Steward, or bringing it back? Upload a + steward-<time>.tar instead of creating an account: its users, settings, hosts, + monitors and key come back, and you sign in as one of its users.

+ {% if restore_error %}

{{ restore_error }}

{% endif %} +
{{ csrf_field() }} + + +
+
{% endblock %} diff --git a/tests/test_auth.py b/tests/test_auth.py index 99069ed..d4bc16f 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -65,3 +65,71 @@ async def test_require_role_redirects_unauthenticated(client): response = await client.get("/") assert response.status_code == 302 assert "/login" in response.headers["Location"] + + +def _setup_restore_stubs(monkeypatch, tmp_path, schema="same", unopened=0): + """First-run setup's restore (#5818) with the check and the restore + stubbed: restoring.py and check.py have their own tests.""" + from steward.backup import job, restoring + from steward.backup.check import AreaCheck, CheckReport, TableCheck + seen = [] + + async def check_file(app, path, name=""): + return CheckReport(name=name, manifest={}, schema=schema, backup_heads=[], here_heads=[], + key_same=False, sealed=1, unopened=unopened, + areas=[AreaCheck("Settings", [TableCheck("app_settings", same=0)])]) + + def start_restore(app, path, name, **kw): + seen.append({"name": name, "data": path.read_bytes()} | kw) + return True + + monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path) + monkeypatch.setattr(job, "check_file", check_file) + monkeypatch.setattr(restoring, "start_restore", start_restore) + return seen + + +async def _send_backup(client, name="steward-20261015T030000Z.tar"): + import io + + from quart.datastructures import FileStorage + headers = await with_csrf(client) + return await client.post("/setup/restore", headers=headers, files={ + "file": FileStorage(io.BytesIO(b"a backup"), filename=name)}) + + +@pytest.mark.asyncio +async def test_setup_restores_a_backup_without_a_safety_backup(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path, schema="older") + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + response = await _send_backup(client) + assert response.status_code == 302 and response.headers["Location"].endswith("/setup") + [started] = seen + assert started["name"] == "steward-20261015T030000Z.tar" and started["data"] == b"a backup" + assert started["safety"] is False and started["replace_all"] is True + assert started["username"] == "first-run setup" + + +@pytest.mark.asyncio +async def test_setup_refuses_a_backup_it_cant_restore(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path, schema="newer") + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + response = await _send_backup(client) + assert response.status_code == 400 and not seen + assert "update Steward, then restore it" in await response.get_data(as_text=True) + assert not list((tmp_path / "incoming").iterdir()) # nothing left held + + +@pytest.mark.asyncio +async def test_setup_restore_is_gone_once_a_user_exists(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path) + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=1)): + response = await _send_backup(client) + assert response.status_code == 302 and "/login" in response.headers["Location"] and not seen + + +@pytest.mark.asyncio +async def test_setup_offers_a_restore(client): + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + page = await (await client.get("/setup")).get_data(as_text=True) + assert 'action="/setup/restore"' in page and 'enctype="multipart/form-data"' in page diff --git a/tests/test_breadcrumbs.py b/tests/test_breadcrumbs.py index 699b33c..b9d5777 100644 --- a/tests/test_breadcrumbs.py +++ b/tests/test_breadcrumbs.py @@ -33,6 +33,7 @@ TOP_LEVEL = { "steward/templates/errors/404.html", "steward/templates/errors/500.html", "steward/templates/errors/database_unavailable.html", + "steward/templates/errors/restoring.html", } _EXTENDS_BASE = re.compile(r"""\{%-?\s*extends\s+["']base\.html["']""")