"""Making one backup file: steward-.tar holding steward.dump pg_dump --format=custom; excluded tables' rows left out secret.key the key Steward's stored secrets are sealed with manifest.json what this is: Steward version, migration heads, time, the tables whose rows were left out The secret key is in the file by the operator's choice (System 180), so a restore is complete on its own: treat every copy as sensitive. """ from __future__ import annotations import asyncio import io import json import os import tarfile import tempfile from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from sqlalchemy.engine import make_url from .contents import EXCLUDED FORMAT = 1 DUMP_NAME = "steward.dump" KEY_NAME = "secret.key" MANIFEST_NAME = "manifest.json" # A dump past this is given up: Steward's configuration is megabytes. DUMP_DEADLINE_SECONDS = 15 * 60 class BackupError(Exception): """Why a backup couldn't be made, said to the person.""" @dataclass class Made: path: Path name: str size: int manifest: dict def file_name(at: datetime) -> str: return f"steward-{at.astimezone(timezone.utc):%Y%m%dT%H%M%SZ}.tar" def pg_env(database_url: str) -> dict[str, str]: """libpq's environment for the app's database URL, so the password is never on a command line.""" url = make_url(database_url) env = {k: v for k, v in os.environ.items() if not k.startswith("PG")} pairs = {"PGHOST": url.host, "PGPORT": url.port, "PGUSER": url.username, "PGPASSWORD": url.password, "PGDATABASE": url.database} env.update({k: str(v) for k, v in pairs.items() if v not in (None, "")}) ssl = url.query.get("ssl") or url.query.get("sslmode") if isinstance(ssl, str) and ssl: env["PGSSLMODE"] = {"true": "require", "false": "disable"}.get(ssl.lower(), ssl) return env def dump_command(path: Path) -> list[str]: args = ["pg_dump", "--format=custom", "--no-owner", "--no-privileges", f"--file={path}"] args += [f"--exclude-table-data={t}" for t in sorted(EXCLUDED)] return args async def run_dump(database_url: str, path: Path) -> None: try: proc = await asyncio.create_subprocess_exec( *dump_command(path), env=pg_env(database_url), stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) except FileNotFoundError: raise BackupError("pg_dump isn't installed in this image.") from None try: _, err = await asyncio.wait_for(proc.communicate(), DUMP_DEADLINE_SECONDS) except TimeoutError: proc.kill() await proc.wait() raise BackupError(f"pg_dump didn't finish within {DUMP_DEADLINE_SECONDS // 60} minutes.") from None if proc.returncode != 0: lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()] raise BackupError(lines[-1] if lines else f"pg_dump exited {proc.returncode}.") def manifest(at: datetime, heads: list[str]) -> dict: from steward import __version__ return {"format": FORMAT, "steward_version": __version__, "created_at": at.isoformat(), "migration_heads": sorted(heads), "rows_left_out": sorted(EXCLUDED), "contains_secret_key": True} def pack(out: Path, dump: Path, secret_key: str, info: dict) -> None: """The tar, written next to `out` and renamed into place.""" part = out.with_name(out.name + ".part") with tarfile.open(part, "w") as tar: tar.add(dump, arcname=DUMP_NAME) for name, data in ((KEY_NAME, secret_key.encode()), (MANIFEST_NAME, json.dumps(info, indent=2).encode())): member = tarfile.TarInfo(name) member.size = len(data) member.mode = 0o600 member.mtime = int(datetime.now(timezone.utc).timestamp()) tar.addfile(member, io.BytesIO(data)) part.chmod(0o600) part.replace(out) async def make_backup(database_url: str, secret_key: str, heads: list[str], workdir: Path, now: datetime | None = None) -> Made: """Dump the database and pack the backup file into `workdir`.""" at = now or datetime.now(timezone.utc) if not secret_key: raise BackupError("Steward has no secret key to put in the backup.") out = workdir / file_name(at) with tempfile.TemporaryDirectory(dir=workdir) as tmp: dump = Path(tmp) / DUMP_NAME await run_dump(database_url, dump) info = manifest(at, heads) await asyncio.to_thread(pack, out, dump, secret_key, info) return Made(out, out.name, out.stat().st_size, info) def read_backup(path: Path, into: Path) -> tuple[dict, Path, str]: """Unpack a backup file into `into`: (manifest, dump path, secret key). Refuses anything that isn't one.""" try: tar = tarfile.open(path, "r") except (tarfile.TarError, OSError) as exc: raise BackupError(f"{path} isn't a Steward backup: {exc}.") from None with tar: names = set(tar.getnames()) if not {DUMP_NAME, KEY_NAME, MANIFEST_NAME} <= names: raise BackupError(f"{path} isn't a Steward backup: it lacks " f"{', '.join(sorted({DUMP_NAME, KEY_NAME, MANIFEST_NAME} - names))}.") info = json.loads(tar.extractfile(MANIFEST_NAME).read()) if info.get("format") != FORMAT: raise BackupError(f"{path} is backup format {info.get('format')}; this Steward reads {FORMAT}.") key = tar.extractfile(KEY_NAME).read().decode().strip() dump = into / DUMP_NAME with tar.extractfile(DUMP_NAME) as src, open(dump, "wb") as dst: while chunk := src.read(1 << 20): dst.write(chunk) return info, dump, key