FabledCurator/tests/test_backup_service.py

"""FC-3h: backup_service unit tests.

Subprocess calls (pg_dump, tar, psql) are monkeypatched so tests run
without external binaries. The real subprocess behavior is exercised
implicitly via the Celery task tests in test_tasks_backup.py.
"""
import json
import re
import subprocess
from pathlib import Path

import pytest

from backend.app.services import backup_service

pytestmark = pytest.mark.integration


@pytest.fixture
def fake_subprocess(monkeypatch):
    """Fake both subprocess.run (restore path) AND subprocess.Popen (the
    bounded-kill backup path) so tests run without external binaries. Each
    writes the target sentinel and records the cmd in a shared list."""
    calls = []

    class _FakeProc:
        returncode = 0
        stdout = b""
        stderr = b""

    def _write_sentinel(cmd):
        if "pg_dump" in cmd:
            i = cmd.index("-f")
            Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
        elif cmd[0] == "tar" and "-cf" in cmd:
            i = cmd.index("-cf")
            Path(cmd[i + 1]).write_bytes(b"fake tar payload")

    def _fake_run(cmd, **kwargs):
        calls.append(list(cmd))
        _write_sentinel(cmd)
        return _FakeProc()

    class _FakePopen:
        def __init__(self, cmd, **kwargs):
            calls.append(list(cmd))
            self.returncode = 0
            _write_sentinel(cmd)

        def communicate(self, timeout=None):
            return (b"", b"")

        def kill(self):
            self.returncode = -9

    monkeypatch.setattr("subprocess.run", _fake_run)
    monkeypatch.setattr("subprocess.Popen", _FakePopen)
    return calls


class _Snapshot:
    """Stands in for the snapshot a dump is taken from."""
    id = "00000003-0000001B-1"
    COUNTS = {"alembic": "0121", "tables": {"credential": 0, "image_record": 2}}

    def counts(self):
        return self.COUNTS


@pytest.fixture(autouse=True)
def fake_snapshot(monkeypatch):
    from contextlib import contextmanager

    @contextmanager
    def _snapshot(db_url):
        yield _Snapshot()
    monkeypatch.setattr(backup_service, "dump_snapshot", _snapshot)


# --- backup_db -------------------------------------------------------


def test_a_dump_is_taken_from_the_snapshot_its_row_counts_come_from(tmp_path, fake_subprocess):
    result = backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
    assert f"--snapshot={_Snapshot.id}" in fake_subprocess[0]
    assert result["counts"] == _Snapshot.COUNTS
    manifest = json.loads(Path(result["manifest_path"]).read_text())
    assert manifest["counts"] == _Snapshot.COUNTS


def test_backup_db_writes_sql_and_manifest(tmp_path, fake_subprocess):
    result = backup_service.backup_db(
        db_url="postgresql://test@x/db",
        images_root=tmp_path,
        tag="pre-cutover",
        triggered_by="manual",
    )
    sql = Path(result["sql_path"])
    manifest = Path(result["manifest_path"])
    assert sql.is_file() and sql.suffix == ".dump"
    assert manifest.is_file() and manifest.suffix == ".json"
    assert result["kind"] == "db"
    assert result["tar_path"] is None
    assert result["size_bytes"] == len(b"-- fake pg_dump\n")

    parsed = json.loads(manifest.read_text())
    assert parsed["kind"] == "db"
    assert parsed["tag"] == "pre-cutover"
    assert parsed["triggered_by"] == "manual"
    assert parsed["artifact_path"] == str(sql)


def test_backup_db_strips_sqlalchemy_psycopg_driver(tmp_path, fake_subprocess):
    backup_service.backup_db(
        db_url="postgresql+psycopg://u:p@h/d", images_root=tmp_path,
    )
    cmd = fake_subprocess[0]
    assert "pg_dump" in cmd
    assert cmd[-1].startswith("postgresql://")
    assert "+psycopg" not in cmd[-1]


def test_backup_db_uses_compressed_custom_format(tmp_path, fake_subprocess):
    # -Fc → pg_restore-loadable, compressed dump (#739 backup polish).
    backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
    assert "-Fc" in fake_subprocess[0]


def test_backup_db_strips_asyncpg_driver(tmp_path, fake_subprocess):
    backup_service.backup_db(
        db_url="postgresql+asyncpg://u:p@h/d", images_root=tmp_path,
    )
    assert "+asyncpg" not in fake_subprocess[0][-1]


def test_backup_db_default_tag_is_none(tmp_path, fake_subprocess):
    result = backup_service.backup_db(
        db_url="postgresql://u@h/d", images_root=tmp_path,
    )
    parsed = json.loads(Path(result["manifest_path"]).read_text())
    assert parsed["tag"] is None


# --- backup_images ---------------------------------------------------


def test_backup_images_writes_tar_and_manifest(tmp_path, fake_subprocess):
    result = backup_service.backup_images(
        images_root=tmp_path, tag="monthly", triggered_by="manual",
    )
    tar = Path(result["tar_path"])
    assert tar.is_file() and tar.name.endswith(".tar.zst")
    assert result["sql_path"] is None
    assert result["size_bytes"] == len(b"fake tar payload")


def test_backup_images_excludes_backups_and_quarantine(tmp_path, fake_subprocess):
    backup_service.backup_images(images_root=tmp_path)
    cmd = fake_subprocess[0]
    excludes = [arg for arg in cmd if arg.startswith("--exclude=")]
    assert any("_backups" in e for e in excludes)
    assert any("_quarantine" in e for e in excludes)


def test_backup_images_excludes_credentials(tmp_path, fake_subprocess):
    """#4234: the images tarball carried `secrets/credential_key.b64` — the key
    that decrypts the stored session cookies — and `cookies/` itself. A media
    archive must not be a credential leak; encryption at rest is worth nothing
    if the key travels with the data."""
    backup_service.backup_images(images_root=tmp_path)
    excludes = [a for a in fake_subprocess[0] if a.startswith("--exclude=")]
    assert any(e.endswith("/secrets") for e in excludes)
    assert any(e.endswith("/cookies") for e in excludes)


def test_backup_images_excludes_are_root_relative(tmp_path, fake_subprocess):
    """tar matches --exclude against the archived path, which is prefixed with
    the root's own directory name (`-C <parent> <name>`). A bare `secrets`
    would also match an ARTIST folder called secrets; the prefix is what keeps
    the exclusion to the top level."""
    backup_service.backup_images(images_root=tmp_path)
    excludes = [a for a in fake_subprocess[0] if a.startswith("--exclude=")]
    assert excludes and all(
        e.startswith(f"--exclude={tmp_path.name}/") for e in excludes
    )


# --- restore_db ------------------------------------------------------


def test_restore_db_drops_schema_then_loads(tmp_path, fake_subprocess):
    dump_path = tmp_path / "fake.dump"
    dump_path.write_bytes(b"\x00fake custom dump")
    backup_service.restore_db(
        db_url="postgresql://u@h/d", sql_path=dump_path,
    )
    # Two calls: psql -c (DROP SCHEMA), then pg_restore -d (load the dump).
    assert len(fake_subprocess) == 2
    assert fake_subprocess[0][0] == "psql"
    assert "-c" in fake_subprocess[0]
    assert "DROP SCHEMA IF EXISTS public CASCADE" in fake_subprocess[0][-1]
    assert "pg_terminate_backend" in fake_subprocess[0][-1]
    assert "pg_restore" in fake_subprocess[1]
    assert "-d" in fake_subprocess[1]
    assert str(dump_path) in fake_subprocess[1]


def _pg_restore_says(monkeypatch, stderr_text: bytes):
    class _Done:
        returncode = 1

    class _Ok:
        returncode = 0

    def run(cmd, stderr=None, **kw):
        if "pg_restore" in cmd:
            stderr.write(stderr_text)
            return _Done()
        return _Ok()

    monkeypatch.setattr("subprocess.run", run)


_NEWER_CLIENT = (
    b'pg_restore: error: could not execute query: ERROR:  unrecognized configuration '
    b'parameter "transaction_timeout"\nCommand was: SET transaction_timeout = 0;\n'
)


def test_a_newer_pg_restore_setting_what_the_server_lacks_is_not_a_failure(
    tmp_path, monkeypatch,
):
    _pg_restore_says(monkeypatch, _NEWER_CLIENT
                     + b"pg_restore: warning: errors ignored on restore: 1\n")
    backup_service.restore_db(db_url="postgresql://u@h/d", sql_path=tmp_path / "x.dump")


def test_a_real_error_beside_the_harmless_one_fails_the_restore_with_its_words(
    tmp_path, monkeypatch,
):
    _pg_restore_says(monkeypatch, _NEWER_CLIENT
                     + b'pg_restore: error: could not execute query: ERROR:  relation '
                       b'"tag" does not exist\nCommand was: COPY tag FROM stdin;\n'
                     + b"pg_restore: warning: errors ignored on restore: 2\n")
    with pytest.raises(backup_service.RestoreFailed, match='relation "tag" does not exist'):
        backup_service.restore_db(db_url="postgresql://u@h/d", sql_path=tmp_path / "x.dump")


def test_a_failed_restore_says_what_pg_restore_said(tmp_path, monkeypatch):
    class _Done:
        def __init__(self, code):
            self.returncode = code

    def run(cmd, stderr=None, **kw):
        if "pg_restore" in cmd:
            stderr.write(b"pg_restore: processing data\npg_restore: error: disk full\n")
            return _Done(1)
        return _Done(0)

    monkeypatch.setattr("subprocess.run", run)
    with pytest.raises(backup_service.RestoreFailed, match="error: disk full"):
        backup_service.restore_db(db_url="postgresql://u@h/d", sql_path=tmp_path / "x.dump")


def test_every_dump_leaves_out_account_logins(tmp_path, fake_subprocess):
    backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
    assert "--exclude-table-data=credential" in fake_subprocess[0]


# --- restore_images --------------------------------------------------


def test_restore_images_untar_to_parent(tmp_path, fake_subprocess):
    tar_path = tmp_path / "fake.tar.zst"
    tar_path.write_bytes(b"fake")
    backup_service.restore_images(images_root=tmp_path, tar_path=tar_path)
    cmd = fake_subprocess[0]
    assert cmd[:3] == ["tar", "--zstd", "-xf"]
    assert str(tar_path) in cmd
    assert "-C" in cmd


# --- unlink ----------------------------------------------------------


def test_unlink_removes_present_files_and_reports(tmp_path):
    sql = tmp_path / "x.sql"
    sql.write_bytes(b"x")
    tar = tmp_path / "x.tar.zst"
    tar.write_bytes(b"x")
    manifest = tmp_path / "x.json"
    manifest.write_text("{}")
    result = backup_service.unlink_artifact_files(
        sql_path=str(sql), tar_path=str(tar), manifest_path=str(manifest),
    )
    assert result == {"sql": True, "tar": True, "manifest": True}
    assert not sql.exists() and not tar.exists() and not manifest.exists()


def test_unlink_missing_files_returns_true(tmp_path):
    """missing_ok semantics: a non-existent file isn't an error."""
    result = backup_service.unlink_artifact_files(
        sql_path=str(tmp_path / "nope.sql"),
        tar_path=None, manifest_path=None,
    )
    assert result == {"sql": True}


def test_unlink_skips_none_paths():
    """A None path is skipped — not added to the result dict."""
    result = backup_service.unlink_artifact_files(
        sql_path=None, tar_path=None, manifest_path=None,
    )
    assert result == {}


# --- helpers ---------------------------------------------------------


def test_libpq_url_strips_each_known_driver():
    assert backup_service._libpq_url(
        "postgresql+psycopg://u@h/d"
    ) == "postgresql://u@h/d"
    assert backup_service._libpq_url(
        "postgresql+asyncpg://u@h/d"
    ) == "postgresql://u@h/d"
    assert backup_service._libpq_url(
        "postgresql+psycopg2://u@h/d"
    ) == "postgresql://u@h/d"
    # Plain URL passes through unchanged.
    assert backup_service._libpq_url(
        "postgresql://u@h/d"
    ) == "postgresql://u@h/d"


def test_backups_dir_created_on_first_use(tmp_path):
    d = backup_service._backups_dir(tmp_path)
    assert d.is_dir()
    assert d.name == "_backups"


# --- bounded-kill + local-temp (FC #739) -----------------------------


def test_backup_db_dumps_to_local_temp_not_nfs_backups_dir(tmp_path, fake_subprocess):
    """pg_dump must target a LOCAL temp path, not the (NFS) _backups dir —
    so its long phase can't hang uninterruptibly on an NFS write."""
    backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
    cmd = fake_subprocess[0]
    dump_target = cmd[cmd.index("-f") + 1]
    assert "_backups" not in dump_target
    # The finished file still ends up in _backups (moved there).
    result = backup_service.backup_db(
        db_url="postgresql://u@h/d", images_root=tmp_path,
    )
    assert "_backups" in result["sql_path"]


def test_run_bounded_fails_fast_when_unkillable(monkeypatch):
    """A child stuck in D-state (communicate keeps timing out even after kill)
    must NOT block the reaper — _run_bounded kills then re-raises promptly."""
    killed = {"n": 0}

    class _Hang:
        def __init__(self, cmd, **kwargs):
            pass

        def communicate(self, timeout=None):
            raise subprocess.TimeoutExpired(cmd="x", timeout=timeout or 0)

        def kill(self):
            killed["n"] += 1

    monkeypatch.setattr("subprocess.Popen", _Hang)
    with pytest.raises(subprocess.TimeoutExpired):
        backup_service._run_bounded(["pg_dump"], 1)
    assert killed["n"] == 1


# --- backup_db_offsite: pg_dump streamed to the bucket ---------------


class _Dump:
    """A pg_dump stand-in: `output` on stdout, `said` on stderr, exiting with
    `code` once its output has been read."""

    def __init__(self, output=b"", said=b"", code=0):
        self.output, self.said, self.code = output, said, code
        self.cmd = None
        self.killed = False

    def __call__(self, cmd, stdout=None, stderr=None, **kwargs):
        import io
        self.cmd = list(cmd)
        self.args = cmd
        self.stdout = io.BytesIO(self.output)
        stderr.write(self.said)
        self.returncode = None
        return self

    def poll(self):
        return self.returncode

    def wait(self, timeout=None):
        if self.returncode is None:
            self.returncode = -9 if self.killed else self.code
        return self.returncode

    def kill(self):
        self.killed = True


class _Bucket:
    """Records what an Upload was asked to do instead of sending it."""

    def __init__(self, fail_on_part=None):
        self.fail_on_part = fail_on_part
        self.uploads = []

    def __call__(self, target, key, **kwargs):
        from backend.app.services.backup_s3 import S3Error
        bucket = self

        class _Upload:
            def __init__(self):
                self.key, self.received = key, []
                self.completed = self.aborted = False
                bucket.uploads.append(self)

            @property
            def parts(self):
                return len(self.received)

            @property
            def size(self):
                return sum(len(p) for p in self.received)

            def add(self, data):
                if bucket.fail_on_part == len(self.received) + 1:
                    raise S3Error("HTTP 503: busy.")
                self.received.append(data)

            def complete(self):
                self.completed = True

            def abort(self):
                self.aborted = True

        return _Upload()


@pytest.fixture
def offsite(monkeypatch):
    from backend.app.services import backup_s3
    monkeypatch.setattr(backup_s3, "PART_SIZE", 4)
    bucket = _Bucket()
    monkeypatch.setattr(backup_s3, "Upload", bucket)
    target = backup_s3.S3Target("http://archives.example:8081", "curator", "AK", "SK")
    return bucket, target


def test_a_dump_streams_to_the_bucket_in_parts_with_nothing_kept_here(offsite, monkeypatch):
    bucket, target = offsite
    dump = _Dump(output=b"0123456789")
    monkeypatch.setattr("subprocess.Popen", dump)
    result = backup_service.backup_db_offsite(
        db_url="postgresql+psycopg://u:p@h/d", target=target, limit_mbps=50,
    )
    (upload,) = bucket.uploads
    assert upload.received == [b"0123", b"4567", b"89"]
    assert upload.completed and not upload.aborted
    assert re.fullmatch(r"database/fabledcurator-\d{8}T\d{6}Z\.dump", result["key"])
    assert result["key"] == upload.key
    assert (result["bucket"], result["size_bytes"], result["parts"]) == ("curator", 10, 3)
    assert dump.cmd[0] == "nice" and "pg_dump" in dump.cmd and "-Fc" in dump.cmd
    assert "-f" not in dump.cmd                  # stdout, not a file
    assert dump.cmd[-1] == "postgresql://u:p@h/d"
    assert f"--snapshot={_Snapshot.id}" in dump.cmd
    assert result["counts"] == _Snapshot.COUNTS


def test_a_failed_dump_leaves_nothing_in_the_bucket_and_says_why(offsite, monkeypatch):
    bucket, target = offsite
    said = b"pg_dump: dumping contents of table x\npg_dump: error: connection lost\n"
    monkeypatch.setattr("subprocess.Popen", _Dump(output=b"012345", said=said, code=1))
    with pytest.raises(backup_service.DumpFailed, match="^pg_dump: error: connection lost$"):
        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
    (upload,) = bucket.uploads
    assert upload.aborted and not upload.completed


def test_a_bucket_that_stops_taking_parts_stops_the_dump(offsite, monkeypatch):
    from backend.app.services.backup_s3 import S3Error
    bucket, target = offsite
    bucket.fail_on_part = 2
    dump = _Dump(output=b"0123456789")
    monkeypatch.setattr("subprocess.Popen", dump)
    with pytest.raises(S3Error):
        backup_service.backup_db_offsite(db_url="postgresql://u@h/d", target=target)
    assert dump.killed
    (upload,) = bucket.uploads
    assert upload.aborted and not upload.completed


def test_dump_names_sort_by_time():
    assert backup_service.offsite_dump_key("20261107T030000Z") == (
        "database/fabledcurator-20261107T030000Z.dump"
    )