diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 3a70543..1e0ab00 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -240,6 +240,12 @@ jobs: # (last-seen times, backup history) left out of the comparison. docker exec -u app "$SMOKE-app" sh -c 'steward check-backup /tmp/backups/steward-*.tar' | tee /tmp/smoke-check.txt grep -q "Matches what is in place" /tmp/smoke-check.txt + # Put back in place over a changed setting (#5815): the setting comes + # back, and history the backup leaves out (the audit log) stays. + sql -d steward -c "UPDATE app_settings SET value_json = '\"changed\"' WHERE key = 'smoke.backup'" + docker exec -u app "$SMOKE-app" sh -c 'steward restore --in-place /tmp/backups/steward-*.tar' + test "$(sql -d steward -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"' + test "$(sql -d steward -c "SELECT count(*) FROM audit_events WHERE id = 'smoke-audit'")" = "1" sql -d steward -c "CREATE DATABASE restored" docker exec -u app \ -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \ diff --git a/docs/core/backups.md b/docs/core/backups.md index 3a34c05..3822795 100644 --- a/docs/core/backups.md +++ b/docs/core/backups.md @@ -1,8 +1,9 @@ # Backups: Steward backing itself up, and coming back Steward sends a backup of itself to an S3 bucket once a day. Fabled Archives -is the intended target. A backup is one file, and `steward restore` brings -Steward back from it into an empty database. +is the intended target. A backup is one file. Settings → Backups restores it +on a running Steward, first-run setup restores it on a new one, and +`steward restore` brings Steward back from it when Steward can't start. ## What a backup holds @@ -126,8 +127,57 @@ as Steward itself. ## Restoring +### From the web UI + +**On a running Steward: Settings → Backups.** Press **Restore** on one of +the recent backups, or choose a file on the "Check or restore a file" card. +Steward first shows what the backup would change: the same comparison as +Check. Then it says what a restore does and asks you to type `restore`. + +- **A backup of this Steward's schema goes back in place.** Settings, + users, hosts, monitors, alert rules and dashboards become the backup's. + The audit log and the history of everything still here stay. History of + anything the backup doesn't have goes with it. It is one transaction: if + anything fails part-way, nothing changes, and Backups says why. +- **A backup from an older Steward replaces everything**, history included, + and you tick "Replace everything" as well. Steward brings it up to date + as it starts. +- **A backup from a newer Steward isn't offered.** Update Steward first. + +Before anything changes, Steward makes a **safety backup** of itself as it +stands. It goes to `/data/backups` (the newest five are kept) and to the +bucket, and it is listed under Recent backups as "before restore". A +restore can be undone by restoring that. + +While the restore runs: +- every page says Steward is restoring; +- agents get a 503 and keep their readings until Steward is back; +- scheduled work holds. + +Then Steward restarts (it exits with code 3 so Swarm, compose or systemd +starts it again), and you sign in as a user from the backup. If an older +backup's full replace fails after the database was emptied, Steward starts +at first-run setup, where the safety backup can be uploaded. + +**On a new Steward: first-run setup.** Next to "Create the initial admin +account" is **Or restore from a backup**: upload a backup file instead, +and Steward comes up with its users, settings and key. There is nothing to +keep yet, so no safety backup is made. + +The backup's secret key is put in place as `/data/secret.key`. If +`STEWARD_SECRET_KEY` is set instead, it must equal the backup's key, or the +restore stops before anything changes. + +### From the command line + `steward restore` writes into an **empty** database. Pass `--wipe` to drop -everything in the database's public schema first. +everything in the database's public schema first. Use it when Steward +can't start at all. + +`steward restore --in-place` does what Settings → Backups does for a backup +of the same schema, from the shell: it replaces what the backup carries and +keeps the audit log and history, in one transaction. Restart Steward after +it so the restored settings load. 1. Stop Steward (scale its service to 0) so nothing writes during the restore. 2. Download the backup file from Archives. @@ -159,5 +209,7 @@ Postgres 16 with one setting line dropped (`transaction_timeout`, which 16 doesn't know). CI checks the whole path on every build. The built image backs up its own -database, restores into a second one, checks a setting came back and the -metrics table is empty, then starts Steward on it. +database and checks the backup against it. It changes a setting and puts +the backup back in place: the setting returns and the audit log stays. +Then it restores into a second, empty database, checks a setting came back +and the metrics table and audit log are empty, and starts Steward on it. diff --git a/steward/auth/csrf.py b/steward/auth/csrf.py index d6d8b90..d6b19d4 100644 --- a/steward/auth/csrf.py +++ b/steward/auth/csrf.py @@ -53,6 +53,7 @@ def rotate() -> None: UPLOAD_LIMITS: dict[str, int] = { "/settings/backups/check/file/": 512 * 1024 * 1024, "/settings/backups/restore/file/": 512 * 1024 * 1024, + "/setup/restore": 512 * 1024 * 1024, } diff --git a/steward/auth/routes.py b/steward/auth/routes.py index 066f692..abbaa78 100644 --- a/steward/auth/routes.py +++ b/steward/auth/routes.py @@ -295,6 +295,50 @@ async def setup_post(): return redirect(url_for("dashboard.index")) + +@auth_bp.post("/setup/restore") +@public_route("first-run setup's restore (refuses once a user exists)") +async def setup_restore(): + """Bring a new Steward up from a backup instead of an empty start (#5818): + only while no user exists, so there is nothing here to lose and no + safety backup to take. The upload's size limit is raised in + auth/csrf.py UPLOAD_LIMITS. The restore then runs as from Settings → + Backups (backup/restoring.py) and Steward restarts into the backup's + users.""" + import shutil + from pathlib import Path + + from quart import current_app + from steward.backup import job, restoring + from steward.backup.dump import BackupError + if await get_user_count(current_app) > 0: + return redirect(url_for("auth.login")) + + async def refused(error: str): + return await render_template("auth/setup.html", default_url=request.host_url.rstrip("/"), + restore_error=error), 400 + + upload = (await request.files).get("file") + if upload is None or not upload.filename: + return await refused("Choose a backup file to restore.") + name = Path(upload.filename).name + held = restoring.stage() + await upload.save(held.path) + try: + report = await job.check_file(current_app._get_current_object(), held.path, name) + except BackupError as exc: + shutil.rmtree(held.dir, ignore_errors=True) + return await refused(str(exc)) + if report.schema == "newer" or not report.ok: + shutil.rmtree(held.dir, ignore_errors=True) + return await refused("This Steward is older than the one that made the backup; update Steward, " + "then restore it." if report.schema == "newer" else report.summary()) + if not restoring.start_restore(current_app._get_current_object(), held.path, name, + username="first-run setup", replace_all=report.schema == "older", + owned_dir=held.dir, safety=False): + return await refused("A restore is already under way.") + return redirect(url_for("auth.setup")) + # ── Invites (milestone 496) ────────────────────────────────────────────────── _INVITE_GONE = ("This invite link has expired, been used or been withdrawn. " diff --git a/steward/backup/restoring.py b/steward/backup/restoring.py index 09efb79..aad27a0 100644 --- a/steward/backup/restoring.py +++ b/steward/backup/restoring.py @@ -70,22 +70,24 @@ def is_restoring() -> bool: def start_restore(app, path: Path, name: str, *, username: str, replace_all: bool = False, - owned_dir: Path | None = None) -> bool: + owned_dir: Path | None = None, safety: bool = True) -> bool: """Start restoring `path` in the background; False if a restore is already - under way. `owned_dir`, when given, is removed once it is over.""" + under way. `owned_dir`, when given, is removed once it is over. `safety` + False skips the safety backup: first-run setup has nothing to keep.""" if STATE.active or STATE.restart_requested: return False STATE.active, STATE.name, STATE.by = True, name, username STATE.since, STATE.step, STATE.error = datetime.now(timezone.utc), "Reading the backup", "" task = asyncio.get_running_loop().create_task( - _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir)) + _run(app, path, name, username=username, replace_all=replace_all, owned_dir=owned_dir, + safety=safety)) _tasks.add(task) task.add_done_callback(_tasks.discard) return True async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, - owned_dir: Path | None) -> None: + owned_dir: Path | None, safety: bool = True) -> None: from steward.core.audit import log_audit from .dump import BackupError @@ -97,8 +99,10 @@ async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, if pinned and pinned != key: raise BackupError("STEWARD_SECRET_KEY is set and differs from the backup's key; " "set it to the backup's key first, or its passwords and keys can't be read.") - STATE.step = "Saving a safety backup of Steward as it stands" - safety = await _safety_backup(app) + safety_name = "" + if safety: + STATE.step = "Saving a safety backup of Steward as it stands" + safety_name = await _safety_backup(app) STATE.step = "Restoring" plugin_dirs = [Path(d) for d in app.config["PLUGIN_DIRS"]] if replace_all: @@ -114,7 +118,7 @@ async def _run(app, path: Path, name: str, *, username: str, replace_all: bool, if done.secret_key != app.config["SECRET_KEY"] and not pinned: _write_key(done.secret_key) await log_audit(app, None, username, "backup.restored", entity_type="backup", - entity_id=name, detail=detail | {"safety_backup": safety}) + entity_id=name, detail=detail | ({"safety_backup": safety_name} if safety else {})) logger.warning("Restored %s (%s); restarting", name, detail["mode"]) STATE.step = "Restored. Steward is restarting; sign in again when it is back" _restart_soon() diff --git a/steward/cli.py b/steward/cli.py index 6ed361f..63f817d 100644 --- a/steward/cli.py +++ b/steward/cli.py @@ -125,24 +125,37 @@ def check_backup_command(ctx: click.Context, backup_file: Path) -> None: @click.argument("backup_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.option("--wipe", is_flag=True, default=False, help="Drop everything in the database first. Without it the database must be empty.") +@click.option("--in-place", "in_place", is_flag=True, default=False, + help="Replace what the backup carries in a database of the same schema, keeping " + "the audit log and history (what Settings → Backups does).") @click.pass_context -def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None: +def restore_command(ctx: click.Context, backup_file: Path, wipe: bool, in_place: bool) -> None: """Bring Steward back from a backup file into its (empty) database, and put the backup's secret key in place. Start Steward afterwards: it runs - any migrations the backup is behind on.""" + any migrations the backup is behind on. With --in-place, the database + keeps its history and only what the backup carries is replaced.""" from .backup.dump import BackupError - from .backup.restore import restore + from .backup.restore import restore, restore_in_place from .config import _SECRET_KEY_FILE, _env, load_bootstrap + if wipe and in_place: + raise click.UsageError("--wipe and --in-place don't go together.") boot = load_bootstrap(Path(ctx.obj["config"])) + plugin_dirs = [Path(d) for d in boot["plugin_dirs"]] try: - done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe, - plugin_dirs=[Path(d) for d in boot["plugin_dirs"]])) + if in_place: + done = asyncio.run(restore_in_place(backup_file, boot["database_url"], plugin_dirs=plugin_dirs)) + else: + done = asyncio.run(restore(backup_file, boot["database_url"], wipe=wipe, plugin_dirs=plugin_dirs)) except BackupError as exc: raise click.ClickException(str(exc)) from None made = done.manifest.get("created_at", "?") click.echo(f"Restored the backup made {made} (Steward {done.manifest.get('steward_version', '?')}).") - if done.fixed: + if in_place: + click.echo(f"Replaced {len(done.tables)} tables ({done.rows} rows); the audit log and history stayed.") + if done.fixed: + click.echo(f"Put right {done.fixed} rows that pointed at something the backup doesn't have.") + elif done.fixed: click.echo(f"Cleared {done.fixed} references to rows a backup leaves out (e.g. Ansible runs).") env_key = _env("SECRET_KEY") if env_key: @@ -155,4 +168,5 @@ def restore_command(ctx: click.Context, backup_file: Path, wipe: bool) -> None: _SECRET_KEY_FILE.write_text(done.secret_key) _SECRET_KEY_FILE.chmod(0o600) click.echo(f"Wrote the backup's secret key to {_SECRET_KEY_FILE}.") - click.echo("Start Steward now; it brings the database up to date on start.") + click.echo("Restart Steward now so it loads them." if in_place + else "Start Steward now; it brings the database up to date on start.") diff --git a/steward/templates/auth/setup.html b/steward/templates/auth/setup.html index d5e4929..9e8cf6b 100644 --- a/steward/templates/auth/setup.html +++ b/steward/templates/auth/setup.html @@ -1,3 +1,5 @@ +{# auth/setup.html — first-run setup: create the first admin, or restore a + backup instead (#5818). Only shown while no user exists. #} {% extends "base.html" %} {% block title %}First-Run Setup — Steward{% endblock %} {% block content %} @@ -32,5 +34,19 @@ + + {# Or come back from a backup (#5818): its users, settings and key. #} +
+

Or restore from a backup

+

Moving Steward, or bringing it back? Upload a + steward-<time>.tar instead of creating an account: its users, settings, hosts, + monitors and key come back, and you sign in as one of its users.

+ {% if restore_error %}

{{ restore_error }}

{% endif %} +
{{ csrf_field() }} + + +
+
{% endblock %} diff --git a/tests/test_auth.py b/tests/test_auth.py index 99069ed..d4bc16f 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -65,3 +65,71 @@ async def test_require_role_redirects_unauthenticated(client): response = await client.get("/") assert response.status_code == 302 assert "/login" in response.headers["Location"] + + +def _setup_restore_stubs(monkeypatch, tmp_path, schema="same", unopened=0): + """First-run setup's restore (#5818) with the check and the restore + stubbed: restoring.py and check.py have their own tests.""" + from steward.backup import job, restoring + from steward.backup.check import AreaCheck, CheckReport, TableCheck + seen = [] + + async def check_file(app, path, name=""): + return CheckReport(name=name, manifest={}, schema=schema, backup_heads=[], here_heads=[], + key_same=False, sealed=1, unopened=unopened, + areas=[AreaCheck("Settings", [TableCheck("app_settings", same=0)])]) + + def start_restore(app, path, name, **kw): + seen.append({"name": name, "data": path.read_bytes()} | kw) + return True + + monkeypatch.setattr(restoring, "SAFETY_DIR", tmp_path) + monkeypatch.setattr(job, "check_file", check_file) + monkeypatch.setattr(restoring, "start_restore", start_restore) + return seen + + +async def _send_backup(client, name="steward-20261015T030000Z.tar"): + import io + + from quart.datastructures import FileStorage + headers = await with_csrf(client) + return await client.post("/setup/restore", headers=headers, files={ + "file": FileStorage(io.BytesIO(b"a backup"), filename=name)}) + + +@pytest.mark.asyncio +async def test_setup_restores_a_backup_without_a_safety_backup(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path, schema="older") + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + response = await _send_backup(client) + assert response.status_code == 302 and response.headers["Location"].endswith("/setup") + [started] = seen + assert started["name"] == "steward-20261015T030000Z.tar" and started["data"] == b"a backup" + assert started["safety"] is False and started["replace_all"] is True + assert started["username"] == "first-run setup" + + +@pytest.mark.asyncio +async def test_setup_refuses_a_backup_it_cant_restore(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path, schema="newer") + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + response = await _send_backup(client) + assert response.status_code == 400 and not seen + assert "update Steward, then restore it" in await response.get_data(as_text=True) + assert not list((tmp_path / "incoming").iterdir()) # nothing left held + + +@pytest.mark.asyncio +async def test_setup_restore_is_gone_once_a_user_exists(client, monkeypatch, tmp_path): + seen = _setup_restore_stubs(monkeypatch, tmp_path) + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=1)): + response = await _send_backup(client) + assert response.status_code == 302 and "/login" in response.headers["Location"] and not seen + + +@pytest.mark.asyncio +async def test_setup_offers_a_restore(client): + with patch("steward.auth.routes.get_user_count", new=AsyncMock(return_value=0)): + page = await (await client.get("/setup")).get_data(as_text=True) + assert 'action="/setup/restore"' in page and 'enctype="multipart/form-data"' in page diff --git a/tests/test_breadcrumbs.py b/tests/test_breadcrumbs.py index 699b33c..b9d5777 100644 --- a/tests/test_breadcrumbs.py +++ b/tests/test_breadcrumbs.py @@ -33,6 +33,7 @@ TOP_LEVEL = { "steward/templates/errors/404.html", "steward/templates/errors/500.html", "steward/templates/errors/database_unavailable.html", + "steward/templates/errors/restoring.html", } _EXTENDS_BASE = re.compile(r"""\{%-?\s*extends\s+["']base\.html["']""")