FabledSteward/steward/backup/dump.py

"""Making one backup file: steward-<UTC time>.tar holding

    steward.dump    pg_dump --format=custom; excluded tables' rows left out
    secret.key      the key Steward's stored secrets are sealed with
    manifest.json   what this is: Steward version, migration heads, time,
                    the tables whose rows were left out

The secret key is in the file by the operator's choice (System 180), so a
restore is complete on its own: treat every copy as sensitive.
"""
from __future__ import annotations

import asyncio
import io
import json
import os
import tarfile
import tempfile
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path

from sqlalchemy.engine import make_url

from .contents import EXCLUDED

FORMAT = 1
DUMP_NAME = "steward.dump"
KEY_NAME = "secret.key"
MANIFEST_NAME = "manifest.json"
# A dump past this is given up: Steward's configuration is megabytes.
DUMP_DEADLINE_SECONDS = 15 * 60


class BackupError(Exception):
    """Why a backup couldn't be made, said to the person."""


@dataclass
class Made:
    path: Path
    name: str
    size: int
    manifest: dict


def file_name(at: datetime) -> str:
    return f"steward-{at.astimezone(timezone.utc):%Y%m%dT%H%M%SZ}.tar"


def pg_env(database_url: str) -> dict[str, str]:
    """libpq's environment for the app's database URL, so the password is
    never on a command line."""
    url = make_url(database_url)
    env = {k: v for k, v in os.environ.items() if not k.startswith("PG")}
    pairs = {"PGHOST": url.host, "PGPORT": url.port, "PGUSER": url.username,
             "PGPASSWORD": url.password, "PGDATABASE": url.database}
    env.update({k: str(v) for k, v in pairs.items() if v not in (None, "")})
    ssl = url.query.get("ssl") or url.query.get("sslmode")
    if isinstance(ssl, str) and ssl:
        env["PGSSLMODE"] = {"true": "require", "false": "disable"}.get(ssl.lower(), ssl)
    return env


def dump_command(path: Path) -> list[str]:
    args = ["pg_dump", "--format=custom", "--no-owner", "--no-privileges", f"--file={path}"]
    args += [f"--exclude-table-data={t}" for t in sorted(EXCLUDED)]
    return args


async def run_dump(database_url: str, path: Path) -> None:
    try:
        proc = await asyncio.create_subprocess_exec(
            *dump_command(path), env=pg_env(database_url),
            stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
    except FileNotFoundError:
        raise BackupError("pg_dump isn't installed in this image.") from None
    try:
        _, err = await asyncio.wait_for(proc.communicate(), DUMP_DEADLINE_SECONDS)
    except TimeoutError:
        proc.kill()
        await proc.wait()
        raise BackupError(f"pg_dump didn't finish within {DUMP_DEADLINE_SECONDS // 60} minutes.") from None
    if proc.returncode != 0:
        lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()]
        raise BackupError(lines[-1] if lines else f"pg_dump exited {proc.returncode}.")


def manifest(at: datetime, heads: list[str]) -> dict:
    from steward import __version__
    return {"format": FORMAT, "steward_version": __version__, "created_at": at.isoformat(),
            "migration_heads": sorted(heads), "rows_left_out": sorted(EXCLUDED),
            "contains_secret_key": True}


def pack(out: Path, dump: Path, secret_key: str, info: dict) -> None:
    """The tar, written next to `out` and renamed into place."""
    part = out.with_name(out.name + ".part")
    with tarfile.open(part, "w") as tar:
        tar.add(dump, arcname=DUMP_NAME)
        for name, data in ((KEY_NAME, secret_key.encode()),
                           (MANIFEST_NAME, json.dumps(info, indent=2).encode())):
            member = tarfile.TarInfo(name)
            member.size = len(data)
            member.mode = 0o600
            member.mtime = int(datetime.now(timezone.utc).timestamp())
            tar.addfile(member, io.BytesIO(data))
    part.chmod(0o600)
    part.replace(out)


async def make_backup(database_url: str, secret_key: str, heads: list[str], workdir: Path,
                      now: datetime | None = None) -> Made:
    """Dump the database and pack the backup file into `workdir`."""
    at = now or datetime.now(timezone.utc)
    if not secret_key:
        raise BackupError("Steward has no secret key to put in the backup.")
    out = workdir / file_name(at)
    with tempfile.TemporaryDirectory(dir=workdir) as tmp:
        dump = Path(tmp) / DUMP_NAME
        await run_dump(database_url, dump)
        info = manifest(at, heads)
        await asyncio.to_thread(pack, out, dump, secret_key, info)
    return Made(out, out.name, out.stat().st_size, info)


def read_backup(path: Path, into: Path) -> tuple[dict, Path, str]:
    """Unpack a backup file into `into`: (manifest, dump path, secret key).
    Refuses anything that isn't one."""
    try:
        tar = tarfile.open(path, "r")
    except (tarfile.TarError, OSError) as exc:
        raise BackupError(f"{path} isn't a Steward backup: {exc}.") from None
    with tar:
        names = set(tar.getnames())
        if not {DUMP_NAME, KEY_NAME, MANIFEST_NAME} <= names:
            raise BackupError(f"{path} isn't a Steward backup: it lacks "
                              f"{', '.join(sorted({DUMP_NAME, KEY_NAME, MANIFEST_NAME} - names))}.")
        info = json.loads(tar.extractfile(MANIFEST_NAME).read())
        if info.get("format") != FORMAT:
            raise BackupError(f"{path} is backup format {info.get('format')}; this Steward reads {FORMAT}.")
        key = tar.extractfile(KEY_NAME).read().decode().strip()
        dump = into / DUMP_NAME
        with tar.extractfile(DUMP_NAME) as src, open(dump, "wb") as dst:
            while chunk := src.read(1 << 20):
                dst.write(chunk)
    return info, dump, key