FabledSteward/tests/integration/test_backup_job.py

"""Integration: the backup job records each run against Postgres (#5673).
pg_dump isn't in this lane (the publish smoke step runs the real one), so the
dump and the upload are stubbed; what is real is everything Steward records:
history, the last-success time, metrics, audit, and the sealed secret key.
Requires STEWARD_DATABASE_URL."""
from __future__ import annotations

import asyncio
import os
import time
import uuid
from datetime import datetime, timedelta, timezone

import pytest

pytestmark = pytest.mark.integration


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


async def _configure(app, **over):
    from steward.core.settings import set_setting
    values = {"backup.enabled": True, "backup.s3_endpoint": "http://archives:8081",
              "backup.s3_bucket": "steward", "backup.s3_access_key": "AK",
              "backup.s3_secret_key": "SK", "backup.history": [],
              "backup.last_success_at": "", "backup.last_attempt_at": "", **over}
    async with app.db_sessionmaker() as db:
        async with db.begin():
            for k, v in values.items():
                await set_setting(db, k, v)


def _stub(monkeypatch, *, fail_upload: str = ""):
    from steward.backup import job
    from steward.backup.dump import Made
    from steward.backup.s3 import S3Error
    sent = []

    async def make_backup(database_url, secret_key, heads, workdir, now=None):
        path = workdir / "steward-stub.tar"
        path.write_bytes(b"x" * 4096)
        return Made(path, path.name, 4096, {"migration_heads": heads})

    async def put_file(target, key, path, **kw):
        if fail_upload:
            raise S3Error(fail_upload)
        sent.append((target.bucket, key, path.read_bytes()))

    monkeypatch.setattr(job, "make_backup", make_backup)
    monkeypatch.setattr(job, "put_file", put_file)
    return sent


async def _state(app):
    from sqlalchemy import select
    from steward.core.settings import get_all_settings
    from steward.models.audit import AuditEvent
    from steward.models.metrics import PluginMetricLatest
    async with app.db_sessionmaker() as db:
        settings = await get_all_settings(db)
        metrics = dict((await db.execute(select(PluginMetricLatest.metric_name, PluginMetricLatest.value).where(
            PluginMetricLatest.source_module == "steward",
            PluginMetricLatest.resource_name == "backup"))).all())
        audits = list((await db.execute(select(AuditEvent.action).where(
            AuditEvent.action.like("backup.%")).order_by(AuditEvent.timestamp.desc(), AuditEvent.id.desc()).limit(1))).scalars())
    return settings, metrics, audits


def test_a_sent_backup_is_recorded_and_a_failed_one_says_why(app, monkeypatch):
    from steward.backup import job

    async def _go():
        await _configure(app)
        sent = _stub(monkeypatch)
        entry = await job.run_backup(app, by="admin-x")
        ok_state = await _state(app)

        _stub(monkeypatch, fail_upload="Upload of steward-stub.tar refused: HTTP 403 AccessDenied.")
        failed = await job.run_backup(app)
        bad_state = await _state(app)
        return sent, entry, ok_state, failed, bad_state

    sent, entry, (settings, metrics, audits), failed, (settings2, _, audits2) = asyncio.run(_go())
    assert sent == [("steward", "steward-stub.tar", b"x" * 4096)]
    assert entry["ok"] and entry["by"] == "admin-x" and entry["size"] == 4096
    assert settings["backup.history"][0]["name"] == "steward-stub.tar"
    assert settings["backup.last_success_at"] == entry["at"]
    assert metrics["backup_bytes"] == 4096 and metrics["backup_age_secs"] == 0
    assert audits == ["backup.made"]

    assert not failed["ok"] and "AccessDenied" in failed["error"]
    assert [h["ok"] for h in settings2["backup.history"]] == [False, True]
    assert settings2["backup.last_success_at"] == entry["at"]       # the good one still stands
    assert settings2["backup.last_attempt_at"] == failed["at"]
    assert audits2 == ["backup.failed"]


def test_the_tick_records_the_age_and_backs_up_when_due(app, monkeypatch):
    from steward.backup import job

    async def _go():
        long_ago = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat()
        await _configure(app, **{"backup.last_success_at": long_ago, "backup.last_attempt_at": long_ago})
        sent = _stub(monkeypatch)
        await job.tick(app)
        settings, metrics, _ = await _state(app)
        await job.tick(app)                    # not due again until the next slot
        return sent, settings

    sent, settings = asyncio.run(_go())
    assert len(sent) == 1
    assert settings["backup.history"][0]["by"] == "schedule"


def test_the_tick_does_nothing_while_backups_are_off(app, monkeypatch):
    from steward.backup import job

    async def _go():
        await _configure(app, **{"backup.enabled": False})
        sent = _stub(monkeypatch)
        await job.tick(app)
        return sent

    assert asyncio.run(_go()) == []


def test_the_secret_key_is_sealed_and_a_blank_box_keeps_it(app):
    from sqlalchemy import select
    from steward.models.settings import AppSetting
    from steward.models.users import User, UserRole

    async def _go():
        await _configure(app)
        uid = str(uuid.uuid4())
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(User(id=uid, username=f"bk-{uid[:8]}", email=f"bk-{uid[:8]}@test",
                           password_hash="x", role=UserRole.admin, is_active=True))
        client = app.test_client()
        async with client.session_transaction() as sess:
            sess.update(user_id=uid, user_role="admin", username=f"bk-{uid[:8]}", _csrf="test-token",
                        login_at=time.time(), tz="UTC")
        form = {"csrf_token": "test-token", "backup.enabled": "on", "backup.s3_endpoint": "http://archives:8081",
                "backup.s3_bucket": "steward", "backup.s3_region": "", "backup.s3_access_key": "AK2",
                "backup.s3_secret_key": "new-secret", "backup.schedule_hour": "4"}
        r1 = await client.post("/settings/backups/", form=form)
        async with app.db_sessionmaker() as db:
            stored = (await db.execute(select(AppSetting.value_json).where(
                AppSetting.key == "backup.s3_secret_key"))).scalar()
        r2 = await client.post("/settings/backups/", form=form | {"backup.s3_secret_key": ""})
        page = await client.get("/settings/backups/")
        from steward.core.settings import get_all_settings
        async with app.db_sessionmaker() as db:
            settings = await get_all_settings(db)
        return r1.status_code, stored, r2.status_code, page.status_code, await page.get_data(as_text=True), settings

    s1, stored, s2, s3, page, settings = asyncio.run(_go())
    assert s1 == 302 and s2 == 302 and s3 == 200
    assert "enc:v1:" in stored and "new-secret" not in stored
    assert settings["backup.s3_secret_key"] == "new-secret"
    assert settings["backup.s3_access_key"] == "AK2" and settings["backup.s3_region"] == "us-east-1"
    assert settings["backup.schedule_hour"] == 4
    assert "new-secret" not in page and "kept; type to change" in page