From 319e7de54756af9c2dead995b1bcfa4b10306867 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 24 May 2026 22:52:35 -0400 Subject: [PATCH] =?UTF-8?q?fc3h:=20backup=5Fservice.py=20=E2=80=94=20DB=20?= =?UTF-8?q?+=20images=20backup/restore=20+=20unlink=20helpers=20(relocated?= =?UTF-8?q?,=20split=20per-kind)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/backup_service.py | 196 +++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 backend/app/services/backup_service.py diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py new file mode 100644 index 0000000..cc177b9 --- /dev/null +++ b/backend/app/services/backup_service.py @@ -0,0 +1,196 @@ +"""FC-3h: first-class backup/restore service for FC. + +Two independent backup kinds: + - 'db' — pg_dump only; fast; nightly via Beat (settings-gated) + - 'images' — tar+zstd of /images; slow; manual trigger only + +Files live under /_backups/. Each backup writes: + fc__.{sql|tar.zst} — the artifact + fc__.json — manifest (kind/tag/triggered_by) + +Service functions are sync (subprocess-bound). Celery tasks in +backend.app.tasks.backup wrap each one with task_run-tracked +lifecycle + soft/hard time limits + retention bookkeeping. +""" +from __future__ import annotations + +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +_BACKUPS_DIRNAME = "_backups" + +# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The +# Celery soft limit signals the Python process; subprocess.Popen in a +# blocking syscall ignores that signal. These bound the worst case. +_DB_SUBPROCESS_TIMEOUT_S = 12 * 60 # 12 min (Celery soft is 10 min) +_IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60 # 7 hr (Celery soft is 6 hr) + + +def _libpq_url(sa_url: str) -> str: + """Strip SQLAlchemy +psycopg/+asyncpg driver suffix for pg_dump/psql.""" + for driver in ( + "postgresql+psycopg", + "postgresql+asyncpg", + "postgresql+psycopg2", + ): + if sa_url.startswith(driver + "://"): + return "postgresql://" + sa_url[len(driver) + 3:] + return sa_url + + +def _backups_dir(images_root: Path) -> Path: + p = images_root / _BACKUPS_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p + + +def _now_ts() -> str: + return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + + +def _file_size_or_none(path: Path) -> int | None: + try: + return path.stat().st_size + except OSError: + return None + + +def _write_manifest( + out_dir: Path, *, kind: str, ts: str, + tag: str | None, triggered_by: str, + artifact_path: Path, +) -> Path: + manifest = { + "kind": kind, + "backup_id": f"fc_{kind}_{ts}", + "tag": tag, + "triggered_by": triggered_by, + "created_at": datetime.now(UTC).isoformat(), + "artifact_path": str(artifact_path), + } + mf = out_dir / f"fc_{kind}_{ts}.json" + mf.write_text(json.dumps(manifest, indent=2)) + return mf + + +def backup_db( + *, db_url: str, images_root: Path, + tag: str | None = None, triggered_by: str = "manual", +) -> dict: + """Run pg_dump; write .sql + manifest; return dict for the caller + to persist into BackupRun. Raises on subprocess failure.""" + ts = _now_ts() + out_dir = _backups_dir(images_root) + sql_path = out_dir / f"fc_db_{ts}.sql" + subprocess.run( + [ + "pg_dump", "--no-owner", "--no-acl", + "-f", str(sql_path), _libpq_url(db_url), + ], + capture_output=True, check=True, + timeout=_DB_SUBPROCESS_TIMEOUT_S, + ) + manifest_path = _write_manifest( + out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by, + artifact_path=sql_path, + ) + return { + "kind": "db", + "ts": ts, + "sql_path": str(sql_path), + "tar_path": None, + "manifest_path": str(manifest_path), + "size_bytes": _file_size_or_none(sql_path), + } + + +def backup_images( + *, images_root: Path, + tag: str | None = None, triggered_by: str = "manual", +) -> dict: + """Run tar --zstd over images_root; write .tar.zst + manifest.""" + ts = _now_ts() + out_dir = _backups_dir(images_root) + tar_path = out_dir / f"fc_images_{ts}.tar.zst" + subprocess.run( + [ + "tar", "--zstd", "-cf", str(tar_path), + "-C", str(images_root.parent), images_root.name, + f"--exclude={images_root.name}/_backups", + f"--exclude={images_root.name}/_quarantine", + ], + capture_output=True, check=True, + timeout=_IMAGES_SUBPROCESS_TIMEOUT_S, + ) + manifest_path = _write_manifest( + out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by, + artifact_path=tar_path, + ) + return { + "kind": "images", + "ts": ts, + "sql_path": None, + "tar_path": str(tar_path), + "manifest_path": str(manifest_path), + "size_bytes": _file_size_or_none(tar_path), + } + + +def restore_db(*, db_url: str, sql_path: Path) -> None: + """Wipe public schema, then load from .sql. Raises on subprocess + failure; partial-restore state is the caller's concern.""" + libpq = _libpq_url(db_url) + subprocess.run( + [ + "psql", libpq, "-c", + "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;", + ], + capture_output=True, check=True, timeout=120, + ) + subprocess.run( + ["psql", libpq, "-f", str(sql_path)], + capture_output=True, check=True, + timeout=_DB_SUBPROCESS_TIMEOUT_S, + ) + + +def restore_images(*, images_root: Path, tar_path: Path) -> None: + """Untar over images_root.parent. Additive — files NOT in the + tarball are NOT removed. Caller wipes first if a clean restore + is needed.""" + subprocess.run( + [ + "tar", "--zstd", "-xf", str(tar_path), + "-C", str(images_root.parent), + ], + capture_output=True, check=True, + timeout=_IMAGES_SUBPROCESS_TIMEOUT_S, + ) + + +def unlink_artifact_files( + *, + sql_path: str | None, + tar_path: str | None, + manifest_path: str | None, +) -> dict: + """Best-effort unlink of all on-disk files for a BackupRun row. + Returns dict keyed by label with True/False per file. Missing + files count as success (missing_ok semantics).""" + deleted: dict = {} + for label, p in ( + ("sql", sql_path), + ("tar", tar_path), + ("manifest", manifest_path), + ): + if not p: + continue + path = Path(p) + try: + path.unlink(missing_ok=True) + deleted[label] = True + except OSError: + deleted[label] = False + return deleted