FabledCurator/backend/app/services/backup_service.py

"""FC-3h: first-class backup/restore service for FC.

Two independent backup kinds:
  - 'db'     — pg_dump only; fast; nightly via Beat (settings-gated)
  - 'images' — tar+zstd of /images; slow; manual trigger only

With an off-site destination set up, a database dump streams straight to that
bucket instead (backup_db_offsite). Otherwise files live under
<images_root>/_backups/. Each local backup writes:
  fc_<kind>_<ts>.{sql|tar.zst}        — the artifact
  fc_<kind>_<ts>.json                 — manifest (kind/tag/triggered_by)

Service functions are sync (subprocess-bound). Celery tasks in
backend.app.tasks.backup wrap each one with task_run-tracked
lifecycle + soft/hard time limits + retention bookkeeping.
"""
from __future__ import annotations

import json
import os
import shutil
import subprocess
import tempfile
import time
from datetime import UTC, datetime
from pathlib import Path

_BACKUPS_DIRNAME = "_backups"

# Excluded from the images tarball, and each for its own reason (#4233, #4234):
#
#   _backups    — the archive would otherwise contain every previous archive.
#                 This is not hypothetical: the 2026-05-23/24 runs, taken before
#                 this exclude existed, grew 43G -> 107G -> ... -> 2123G as each
#                 swallowed its predecessors, and cost 4.3T of the images
#                 filesystem until they were reclaimed on 2026-09-21.
#   _quarantine — holds files deliberately pulled OUT of the library.
#   secrets     — `credential_key.b64`, the key that decrypts the stored
#                 Patreon/SubscribeStar session credentials.
#   cookies     — those session cookies themselves.
#
# The last two are the ones worth stating plainly: an images tarball is a media
# archive, and a media archive that carries the key to the operator's accounts
# is a credential leak wearing a backup's name. Encryption at rest buys nothing
# when the key rides along in the same file. A restore therefore does NOT
# re-establish credentials — you sign in again, which is the correct outcome.
_IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")

# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The Celery
# soft limit signals the Python process; subprocess.Popen in a blocking syscall
# ignores that signal, so these bound the worst case directly. Each sits just
# UNDER its task's Celery soft_time_limit so the bounded-kill (_run_bounded) is
# the primary guard and fires cleanly before Celery's soft/hard limits — which
# matters because an NFS D-state hang defeats even Celery's SIGKILL (the failure
# that wedged the maintenance lane for hours, #739).
#   backup_db_task:     soft=1800s / hard=2100s  → 1700s
#   backup_images_task: soft=21600s / hard=23400s → 21000s
_DB_SUBPROCESS_TIMEOUT_S = 1700            # ~28 min, under the 30-min DB soft limit
_IMAGES_SUBPROCESS_TIMEOUT_S = 21000       # ~5.8 hr, under the 6-hr images soft limit
# Grace after SIGKILL to reap the child. If it can't be reaped in this window
# (an uninterruptible NFS D-state — the failure mode that wedged the
# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we
# STOP waiting and fail fast, freeing the worker slot. The orphan is reaped by
# the OS once its blocking syscall clears.
_KILL_REAP_GRACE_S = 10


# Every database dump, local or off-site: custom format (-Fc, compressed and
# restored with pg_restore), without ownership or grants so it loads into any
# install. The `credential` table's rows (account logins, sealed with the key
# in /images/secrets) stay out: a restored install keeps the table, empty, and
# you sign in to each account again.
_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc", "--exclude-table-data=credential"]


def _run_bounded(cmd: list[str], timeout: int) -> None:
    """subprocess.run(check=True, timeout) whose reaper can't itself hang.

    subprocess.run's timeout path SIGKILLs the child then blocks in wait() to
    reap it — but a process stuck in uninterruptible I/O (NFS) can't be reaped,
    so wait() blocks for hours. Here we bound the post-kill reap and re-raise
    TimeoutExpired regardless, so the caller fails fast instead of wedging."""
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    try:
        out, err = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        proc.kill()
        try:
            proc.communicate(timeout=_KILL_REAP_GRACE_S)
        except subprocess.TimeoutExpired:
            pass  # unkillable (D-state) — abandon the reap, fail fast
        raise
    if proc.returncode != 0:
        raise subprocess.CalledProcessError(
            proc.returncode, cmd, output=out, stderr=err
        )


def _libpq_url(sa_url: str) -> str:
    """Strip SQLAlchemy +psycopg/+asyncpg driver suffix for pg_dump/psql."""
    for driver in (
        "postgresql+psycopg",
        "postgresql+asyncpg",
        "postgresql+psycopg2",
    ):
        if sa_url.startswith(driver + "://"):
            return "postgresql://" + sa_url[len(driver) + 3:]
    return sa_url


def _backups_dir(images_root: Path) -> Path:
    p = images_root / _BACKUPS_DIRNAME
    p.mkdir(parents=True, exist_ok=True)
    return p


def _now_ts() -> str:
    return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")


def _file_size_or_none(path: Path) -> int | None:
    try:
        return path.stat().st_size
    except OSError:
        return None


def _write_manifest(
    out_dir: Path, *, kind: str, ts: str,
    tag: str | None, triggered_by: str,
    artifact_path: Path,
) -> Path:
    manifest = {
        "kind": kind,
        "backup_id": f"fc_{kind}_{ts}",
        "tag": tag,
        "triggered_by": triggered_by,
        "created_at": datetime.now(UTC).isoformat(),
        "artifact_path": str(artifact_path),
    }
    mf = out_dir / f"fc_{kind}_{ts}.json"
    mf.write_text(json.dumps(manifest, indent=2))
    return mf


def backup_db(
    *, db_url: str, images_root: Path,
    tag: str | None = None, triggered_by: str = "manual",
) -> dict:
    """Run pg_dump; write .sql + manifest; return dict for the caller
    to persist into BackupRun. Raises on subprocess failure."""
    ts = _now_ts()
    out_dir = _backups_dir(images_root)
    # Custom format (-Fc): compressed (much smaller on NFS) and restored with
    # pg_restore. The .dump extension marks it as non-SQL. The BackupRun field
    # is still named sql_path — it's just "the db artifact path".
    sql_path = out_dir / f"fc_db_{ts}.dump"
    # Dump to LOCAL disk first, then move the finished file to the (NFS) backups
    # dir. pg_dump's long phase is then a DB-socket wait + local writes — both
    # killable — instead of an NFS write that can hang uninterruptibly. Only the
    # final move touches NFS, and it's a bounded single-file step.
    fd, tmp_name = tempfile.mkstemp(prefix="fc_db_", suffix=".dump")
    os.close(fd)
    tmp_path = Path(tmp_name)
    try:
        _run_bounded(
            _gently([*_PG_DUMP, "-f", str(tmp_path), _libpq_url(db_url)]),
            _DB_SUBPROCESS_TIMEOUT_S,
        )
        shutil.move(str(tmp_path), str(sql_path))
    finally:
        if tmp_path.exists():
            tmp_path.unlink(missing_ok=True)
    manifest_path = _write_manifest(
        out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
        artifact_path=sql_path,
    )
    return {
        "kind": "db",
        "ts": ts,
        "sql_path": str(sql_path),
        "tar_path": None,
        "manifest_path": str(manifest_path),
        "size_bytes": _file_size_or_none(sql_path),
    }


class DumpFailed(Exception):
    """pg_dump exited with an error; the message is its last line of output."""


def offsite_dump_key(ts: str) -> str:
    """Where a database dump lands in the off-site bucket. Dated names sit
    side by side under one prefix, which is what lets the bucket's retention
    thin them by age."""
    return f"database/fabledcurator-{ts}.dump"


def _gently(cmd: list[str]) -> list[str]:
    """`cmd` at low CPU priority, and low disk priority where `ionice` is
    installed, so a long dump leaves the server responsive."""
    io = ["ionice", "-c", "2", "-n", "7"] if shutil.which("ionice") else []
    return ["nice", "-n", "10", *io, *cmd]


def _last_line(f) -> str | None:
    f.seek(0)
    lines = f.read().decode(errors="replace").strip().splitlines()
    return lines[-1].strip() if lines else None


def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) -> dict:
    """Stream `pg_dump` straight into the off-site bucket, in parts, with no
    copy staged on this server. The object appears only once pg_dump has
    exited cleanly and every part has arrived; a dump that fails partway
    leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
    last line. Returns a dict for the caller to record in BackupRun."""
    from .backup_s3 import PART_SIZE, Pacer, Upload

    ts = _now_ts()
    key = offsite_dump_key(ts)
    deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
    with tempfile.TemporaryFile() as err:
        proc = subprocess.Popen(
            _gently([*_PG_DUMP, _libpq_url(db_url)]),
            stdout=subprocess.PIPE, stderr=err,
        )
        upload = None
        try:
            upload = Upload(target, key, pacer=Pacer(limit_mbps))
            while part := proc.stdout.read(PART_SIZE):
                if time.monotonic() > deadline:
                    raise subprocess.TimeoutExpired(proc.args, _DB_SUBPROCESS_TIMEOUT_S)
                upload.add(part)
            proc.stdout.close()
            proc.wait(timeout=_KILL_REAP_GRACE_S * 6)
            if proc.returncode != 0:
                raise DumpFailed(_last_line(err) or f"pg_dump exited with {proc.returncode}")
            upload.complete()
        except BaseException:
            if proc.poll() is None:
                proc.kill()
                try:
                    proc.wait(timeout=_KILL_REAP_GRACE_S)
                except subprocess.TimeoutExpired:
                    pass  # unkillable (D-state): abandon the reap, fail fast
            if upload is not None:
                upload.abort()
            raise
    return {
        "kind": "db",
        "ts": ts,
        "key": key,
        "bucket": target.bucket,
        "endpoint": target.endpoint,
        "parts": upload.parts,
        "size_bytes": upload.size,
    }


def backup_images(
    *, images_root: Path,
    tag: str | None = None, triggered_by: str = "manual",
) -> dict:
    """Run tar --zstd over images_root; write .tar.zst + manifest."""
    ts = _now_ts()
    out_dir = _backups_dir(images_root)
    tar_path = out_dir / f"fc_images_{ts}.tar.zst"
    # No local-temp here (the archive is hundreds of GB — it can't stage in
    # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast
    # rather than holding the lane for hours.
    _run_bounded(
        [
            "tar", "--zstd", "-cf", str(tar_path),
            "-C", str(images_root.parent), images_root.name,
            *(
                f"--exclude={images_root.name}/{name}"
                for name in _IMAGES_EXCLUDED_DIRNAMES
            ),
        ],
        _IMAGES_SUBPROCESS_TIMEOUT_S,
    )
    manifest_path = _write_manifest(
        out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
        artifact_path=tar_path,
    )
    return {
        "kind": "images",
        "ts": ts,
        "sql_path": None,
        "tar_path": str(tar_path),
        "manifest_path": str(manifest_path),
        "size_bytes": _file_size_or_none(tar_path),
    }


class RestoreFailed(Exception):
    """psql or pg_restore exited with an error; the message is its last line."""


# A restore waits this long for other connections to let go of the tables it
# replaces, then gives up with Postgres' own reason instead of queueing every
# other query behind it.
_RESTORE_LOCK_TIMEOUT = "60s"
# pg_restore of a large library (vector tables, rebuilt indexes) runs well past
# a dump's length; it is bounded on its own.
_RESTORE_SUBPROCESS_TIMEOUT_S = 3 * 3600


def _run_reporting(cmd: list[str], timeout: int, env: dict | None = None) -> None:
    """Run `cmd`; raise RestoreFailed with the last line it printed when it
    exits non-zero."""
    with tempfile.TemporaryFile() as err:
        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err, timeout=timeout,
                              env=env)
        if proc.returncode != 0:
            raise RestoreFailed(_last_line(err) or f"{cmd[0]} exited with {proc.returncode}")


# Every other connection to this database: the web server's, the other
# workers'. Ended before the tables are dropped, so none holds a lock the drop
# waits on, and again once the restore is in, so none keeps statements it
# prepared against the old tables. Each reconnects on its next query.
_END_OTHER_CONNECTIONS = (
    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
    "WHERE datname = current_database() AND pid <> pg_backend_pid();"
)


def end_other_connections(*, db_url: str) -> None:
    _run_reporting(["psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1",
                    "-c", _END_OTHER_CONNECTIONS], 60)


def wipe_db(*, db_url: str) -> None:
    """Drop and recreate the public schema: an empty database, ready for a
    dump or for `alembic upgrade head`."""
    _run_reporting(
        [
            "psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1", "-c",
            f"{_END_OTHER_CONNECTIONS} DROP SCHEMA IF EXISTS public CASCADE; "
            "CREATE SCHEMA public;",
        ],
        120,
        # Set for the session, not as a SET inside the same -c string.
        env={**os.environ, "PGOPTIONS": f"-c lock_timeout={_RESTORE_LOCK_TIMEOUT}"},
    )


# A pg_restore newer than the server sets parameters the server doesn't have
# (pg_restore 17 against Postgres 16: `SET transaction_timeout = 0`). The
# statement fails and nothing depends on it, so its error is not a failed
# restore.
_HARMLESS_RESTORE_ERRORS = (
    'unrecognized configuration parameter "transaction_timeout"',
)


def _restore_errors(stderr: str) -> list[str]:
    """pg_restore's error lines, less the harmless ones."""
    return [
        line.strip() for line in stderr.splitlines()
        if "error:" in line and not any(h in line for h in _HARMLESS_RESTORE_ERRORS)
    ]


def restore_db(*, db_url: str, sql_path: Path) -> None:
    """Wipe the public schema, then load the custom-format dump. Any error
    pg_restore reports, other than a harmless one, raises RestoreFailed with
    the first; the database is then partly loaded, and the caller decides what
    to put back."""
    wipe_db(db_url=db_url)
    # Custom-format (-Fc) dumps are restored with pg_restore, not psql. It runs
    # on past errors (it can't skip the harmless one and stop at the rest), so
    # its errors are judged once it is done.
    cmd = _gently(["pg_restore", "--no-owner", "--no-acl",
                   "-d", _libpq_url(db_url), str(sql_path)])
    with tempfile.TemporaryFile() as err:
        proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err,
                              timeout=_RESTORE_SUBPROCESS_TIMEOUT_S)
        if proc.returncode == 0:
            return
        err.seek(0)
        said = err.read().decode(errors="replace")
    errors = _restore_errors(said)
    if errors:
        raise RestoreFailed(errors[0] if len(errors) == 1
                            else f"{errors[0]} ({len(errors)} errors in all)")
    if not any(h in said for h in _HARMLESS_RESTORE_ERRORS):
        # A failure with no error line to show for it.
        lines = said.strip().splitlines()
        raise RestoreFailed(lines[-1].strip() if lines
                            else f"pg_restore exited with {proc.returncode}")


def restore_images(*, images_root: Path, tar_path: Path) -> None:
    """Untar over images_root.parent. Additive — files NOT in the
    tarball are NOT removed. Caller wipes first if a clean restore
    is needed."""
    subprocess.run(
        [
            "tar", "--zstd", "-xf", str(tar_path),
            "-C", str(images_root.parent),
        ],
        capture_output=True, check=True,
        timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
    )


def unlink_artifact_files(
    *,
    sql_path: str | None,
    tar_path: str | None,
    manifest_path: str | None,
) -> dict:
    """Best-effort unlink of all on-disk files for a BackupRun row.
    Returns dict keyed by label with True/False per file. Missing
    files count as success (missing_ok semantics)."""
    deleted: dict = {}
    for label, p in (
        ("sql", sql_path),
        ("tar", tar_path),
        ("manifest", manifest_path),
    ):
        if not p:
            continue
        path = Path(p)
        try:
            path.unlink(missing_ok=True)
            deleted[label] = True
        except OSError:
            deleted[label] = False
    return deleted