"""Where the scheduled backup goes, and getting a backup file there. Two kinds of destination, both instance-wide and set by an admin: - **folder**: a directory inside the container, usually a mounted share. A backup service that watches the share (or a person) takes it from there. - **s3**: any S3-compatible bucket, addressed by path. Scribe only puts and lists, so an append-only key is enough. Either way Scribe ADDS one dated file per run and never removes one: keeping and thinning the history is the destination's job (services/backup_file.py says why the file is shaped the way it is for that). The settings live as admin settings: `backup_config` holds everything but the S3 secret as JSON, and `backup_s3_secret` holds the secret sealed under the instance's SECRET_KEY (services/secrets.py). The secret is also kept out of every backup (backup._SETTINGS_NOT_EXPORTED), so a backup copied off-site does not carry the key that writes to that very place. """ from __future__ import annotations import asyncio import json import os import tempfile from dataclasses import asdict, dataclass, fields, replace from datetime import datetime from pathlib import Path from scribe.services import backup_file from scribe.services.backup_s3 import S3Error, S3Target, list_objects, put_file from scribe.services.secrets import seal, unseal from scribe.services.settings import get_admin_setting, set_admin_setting CONFIG_SETTING = "backup_config" SECRET_SETTING = "backup_s3_secret" DESTINATION_KINDS = ("folder", "s3") DEFAULT_FOLDER = "/backups" # 02:00 UTC: before the 03:30 trash purge and the 04:00 DB maintenance, so the # backup never contends with a VACUUM, and the night's deletions are still # recoverable from the backup taken just before them. DEFAULT_HOUR = 2 class DestinationError(Exception): """Why the destination can't take a backup, said to the person.""" @dataclass(frozen=True) class BackupConfig: enabled: bool = False hour: int = DEFAULT_HOUR destination: str = "folder" folder_path: str = DEFAULT_FOLDER s3_endpoint: str = "" s3_bucket: str = "" s3_access_key: str = "" s3_region: str = "us-east-1" s3_verify_tls: bool = True @classmethod def from_dict(cls, raw: dict) -> "BackupConfig": """Lenient: unknown keys are ignored and bad values fall back to the default, so an edited or older stored config still loads.""" known = {f.name: f for f in fields(cls)} values = {} for name, value in (raw or {}).items(): f = known.get(name) if f is None: continue default = f.default if isinstance(default, bool): values[name] = value if isinstance(value, bool) else default elif isinstance(default, int): try: values[name] = int(value) except (TypeError, ValueError): values[name] = default else: values[name] = str(value).strip() if value is not None else default config = cls(**values) if not 0 <= config.hour <= 23: config = replace(config, hour=DEFAULT_HOUR) if config.destination not in DESTINATION_KINDS: config = replace(config, destination="folder") return config def to_dict(self) -> dict: return asdict(self) def problems(self, *, has_secret: bool) -> list[str]: """What stops this config from sending a backup, in words. Empty when it is complete (which says nothing about whether the place answers — check_destination does that).""" out: list[str] = [] if self.destination == "folder": if not self.folder_path.startswith("/"): out.append("The folder must be an absolute path inside the container.") else: if not self.s3_endpoint.startswith(("http://", "https://")): out.append("The S3 address must start with http:// or https://.") if not self.s3_bucket: out.append("The bucket name is missing.") if not self.s3_access_key: out.append("The access key is missing.") if not has_secret: out.append("The secret key is missing.") return out def apply_changes(current: BackupConfig, data: dict) -> BackupConfig: """`current` with the fields an admin sent changed, checked strictly. `from_dict` forgives a stored value it can't read; a value a person just typed is refused instead, with the reason, so a typo in the hour does not silently become 02:00. Keys that aren't config fields (the secret, say) are ignored here. """ names = {f.name for f in fields(BackupConfig)} changes = {k: v for k, v in (data or {}).items() if k in names} if "hour" in changes: try: hour = int(changes["hour"]) except (TypeError, ValueError): raise ValueError("The hour must be a whole number from 0 to 23.") from None if not 0 <= hour <= 23: raise ValueError("The hour must be from 0 to 23 (UTC).") changes["hour"] = hour if "destination" in changes and changes["destination"] not in DESTINATION_KINDS: raise ValueError("The destination must be a folder or an S3 bucket.") for name in ("enabled", "s3_verify_tls"): if name in changes and not isinstance(changes[name], bool): raise ValueError(f"{name} must be true or false.") return BackupConfig.from_dict({**current.to_dict(), **changes}) async def load_config() -> BackupConfig: raw = await get_admin_setting(CONFIG_SETTING, "") try: data = json.loads(raw) if raw else {} except ValueError: data = {} return BackupConfig.from_dict(data if isinstance(data, dict) else {}) async def save_config(config: BackupConfig) -> None: await set_admin_setting(CONFIG_SETTING, json.dumps(config.to_dict(), sort_keys=True)) async def load_secret() -> tuple[str | None, bool]: """(the S3 secret or None, whether one is stored). Stored but None means it was sealed under another SECRET_KEY — after a restore onto a different instance, say — and has to be entered again. """ stored = await get_admin_setting(SECRET_SETTING, "") return (unseal(stored), bool(stored)) async def save_secret(secret: str) -> None: await set_admin_setting(SECRET_SETTING, seal(secret) if secret else "") def s3_target(config: BackupConfig, secret: str) -> S3Target: return S3Target( endpoint=config.s3_endpoint, bucket=config.s3_bucket, access_key=config.s3_access_key, secret_key=secret, region=config.s3_region or "us-east-1", verify_tls=config.s3_verify_tls, ) @dataclass(frozen=True) class Delivered: name: str size_bytes: int where: str async def deliver( payload: dict, config: BackupConfig, secret: str | None, at: datetime, *, app_version: str, transport=None, ) -> Delivered: """Write `payload` as the dated backup file and put it at the destination. The file is written off the event loop: serialising a full export is seconds of CPU on a large corpus, and the app keeps serving meanwhile. """ problems = config.problems(has_secret=bool(secret)) if problems: raise DestinationError(" ".join(problems)) if config.destination == "folder": folder = Path(config.folder_path) if not folder.is_dir(): raise DestinationError( f"{folder} isn't there. Mount the backup share at that path, " "or choose the folder it is mounted at." ) try: path = await asyncio.to_thread( backup_file.write_archive, payload, folder, at, app_version=app_version, ) except PermissionError: raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None return Delivered(path.name, path.stat().st_size, str(folder)) target = s3_target(config, secret or "") with tempfile.TemporaryDirectory(prefix="scribe-backup-") as staging: path = await asyncio.to_thread( backup_file.write_archive, payload, Path(staging), at, app_version=app_version, ) size = path.stat().st_size try: await put_file(target, path.name, path, transport=transport) except S3Error as exc: raise DestinationError(str(exc)) from None return Delivered(path.name, size, target.url()) async def check_destination( config: BackupConfig, secret: str | None, *, transport=None, ) -> str: """Ask the destination whether it would take a backup, without sending one. Returns what was found, in words; raises DestinationError otherwise. The folder check writes and removes a dot-file, which a folder watcher ignores. The S3 check only lists: an append-only key could add a test object and then never remove it. """ problems = config.problems(has_secret=bool(secret)) if problems: raise DestinationError(" ".join(problems)) if config.destination == "folder": folder = Path(config.folder_path) if not folder.is_dir(): raise DestinationError(f"{folder} isn't there, or isn't a folder.") probe = folder / ".scribe-write-check" try: await asyncio.to_thread(probe.write_bytes, b"") await asyncio.to_thread(probe.unlink) except PermissionError: raise DestinationError(f"Scribe isn't allowed to write in {folder}.") from None except OSError as exc: raise DestinationError(f"Writing in {folder} failed: {exc.strerror or exc}.") from None count = await asyncio.to_thread(_count_backups, folder) return f"{folder} is writable and holds {count} Scribe backup(s)." target = s3_target(config, secret or "") try: objects = await list_objects(target, backup_file.NAME_PREFIX, transport=transport) except S3Error as exc: raise DestinationError(str(exc)) from None return f"Bucket {config.s3_bucket} answered and holds {len(objects)} Scribe backup(s)." def _count_backups(folder: Path) -> int: return sum( 1 for name in os.listdir(folder) if name.startswith(backup_file.NAME_PREFIX) and name.endswith(backup_file.NAME_SUFFIX) )