FabledCurator/backend/app/api/system_backup.py

"""/api/system/backup: database dumps, the off-site destination and its media
backup, restores, test restores, and the run history.

Read endpoints are public on FC (operator-facing internal API; same
posture as /api/system/activity). Write endpoints take a typed
`confirm` body field that must match a server-generated token for
that backup row, to prevent click-to-destroy by stale browser tabs
or accidental cURL.
"""
from __future__ import annotations

from quart import Blueprint, jsonify, request
from sqlalchemy import desc, select

from ..extensions import get_session
from ..models import BackupRun, ImportSettings
from ._responses import error_response as _bad

system_backup_bp = Blueprint(
    "system_backup", __name__, url_prefix="/api/system/backup",
)

_KINDS = frozenset({"db", "media", "restore", "reconcile", "test"})
_TAG_MAX_LEN = 64
_BACKUP_SETTINGS_FIELDS = (
    "backup_db_nightly_enabled",
    "backup_db_nightly_hour_utc",
    "backup_db_keep_last_n",
)


def _row_to_dict(r: BackupRun) -> dict:
    return {
        "id": r.id,
        "kind": r.kind,
        "status": r.status,
        "tag": r.tag,
        "triggered_by": r.triggered_by,
        "started_at": r.started_at.isoformat() if r.started_at else None,
        "finished_at": r.finished_at.isoformat() if r.finished_at else None,
        "duration_seconds": (
            int((r.finished_at - r.started_at).total_seconds())
            if r.finished_at and r.started_at else None
        ),
        "sql_path": r.sql_path,
        "size_bytes": r.size_bytes,
        "error": r.error,
        "restored_from_id": r.restored_from_id,
        "manifest": r.manifest or {},
        # Where an off-site dump went ({endpoint, bucket, key}); None for a
        # backup kept on this server.
        "offsite": (r.manifest or {}).get("offsite"),
    }


def _validate_tag(tag):
    if tag is None:
        return None
    if not isinstance(tag, str):
        return _bad("invalid_tag", detail="tag must be string or null")
    tag = tag.strip()
    if not tag:
        return None
    if len(tag) > _TAG_MAX_LEN:
        return _bad("invalid_tag", detail=f"tag too long (max {_TAG_MAX_LEN})")
    return tag


def _validate_backup_settings_patch(body: dict):
    if "backup_db_nightly_enabled" in body and not isinstance(
        body["backup_db_nightly_enabled"], bool,
    ):
        return _bad("invalid_value", detail="backup_db_nightly_enabled must be bool")
    if "backup_db_nightly_hour_utc" in body:
        v = body["backup_db_nightly_hour_utc"]
        if not isinstance(v, int) or isinstance(v, bool) or not (0 <= v <= 23):
            return _bad("invalid_value", detail="backup_db_nightly_hour_utc must be 0..23")
    if "backup_db_keep_last_n" in body:
        v = body["backup_db_keep_last_n"]
        if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
            return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
    return None


@system_backup_bp.route("/db", methods=["POST"])
async def trigger_db_backup():
    body = await request.get_json(silent=True) or {}
    tag = _validate_tag(body.get("tag"))
    if isinstance(tag, tuple):
        return tag
    from ..tasks.backup import backup_db_task
    backup_db_task.delay(tag=tag, triggered_by="manual")
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/media", methods=["POST"])
async def trigger_media_backup():
    """Send the images and attachments not yet in the off-site bucket. Refused
    when no destination is set up; a run already going is left to finish."""
    from ..services import backup_destination as dest

    async with get_session() as session:
        row = await ImportSettings.load(session)
        if not dest.describe(row)["configured"]:
            return _bad("no_destination",
                        detail="set up an off-site destination before sending files")
    from ..tasks.backup import backup_media_task
    backup_media_task.delay(triggered_by="manual")
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/media", methods=["GET"])
async def media_coverage():
    """How much of the library is in the off-site bucket: files and bytes,
    sent and total, each file counted once however many posts hold it."""
    from ..services import backup_destination as dest
    from ..services import backup_media

    async with get_session() as session:
        row = await ImportSettings.load(session)
        f = dest.fields_of(row)
        configured = dest.describe(row)["configured"]
        tid = backup_media.destination_id(f.endpoint, f.bucket) if configured else None
        counts = await session.run_sync(lambda s: backup_media.coverage(s, tid))
    return jsonify({"configured": configured, **counts})


# ── Restoring from the off-site bucket ──────────────────────────────────────


async def _saved_target():
    """(target, None) for the saved destination, or (None, error response)."""
    from ..services import backup_destination as dest
    from ..services.backup_s3 import S3Error

    async with get_session() as session:
        row = await ImportSettings.load(session)
        try:
            target = dest.saved_target(row)
        except S3Error as exc:
            return None, _bad("destination_unreadable", detail=str(exc))
    if target is None:
        return None, _bad("no_destination",
                          detail="set up an off-site destination before restoring from it")
    return target, None


async def _busy_reason() -> str | None:
    from ..tasks.backup import backup_busy

    async with get_session() as session:
        return await session.run_sync(backup_busy)


def restore_token(ts: str) -> str:
    """What the person types to confirm replacing the database with the dump
    taken at `ts`."""
    return f"restore-offsite-{ts}"


@system_backup_bp.route("/offsite/dumps", methods=["GET"])
async def list_offsite_dumps():
    """The database dumps in the off-site bucket, newest first, each with
    the confirmation a restore of it asks for."""
    import asyncio

    from ..services import backup_restore
    from ..services.backup_s3 import S3Error

    target, err = await _saved_target()
    if err:
        return err
    try:
        dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
    except S3Error as exc:
        return _bad("destination_unreachable", detail=str(exc), status=502)
    return jsonify({
        "dumps": [
            {"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "counted": d.counted,
             "confirm": restore_token(d.ts)}
            for d in dumps
        ],
        # A test restore needs about this much free space on the database's
        # disk while it runs.
        "database_bytes": await _database_bytes(),
    })


async def _database_bytes() -> int:
    from sqlalchemy import text

    async with get_session() as session:
        return (await session.execute(
            text("SELECT pg_database_size(current_database())"))).scalar_one()


async def _restore_test_blocked() -> str | None:
    from ..tasks.backup import restore_test_blocked

    async with get_session() as session:
        return await session.run_sync(restore_test_blocked)


@system_backup_bp.route("/offsite/test", methods=["POST"])
async def restore_test_offsite():
    """Test a dump from the bucket without touching this install: it is
    loaded into a temporary database, checked table by table, and dropped.
    Body: {key}, a dump's key from /offsite/dumps, or {} for the newest.
    Needs no confirmation; refused while a restore or another test is going."""
    body = await request.get_json(silent=True) or {}
    key = body.get("key")
    if key is not None and (not isinstance(key, str) or not key.startswith("database/")):
        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
    _, err = await _saved_target()
    if err:
        return err
    busy = await _restore_test_blocked()
    if busy:
        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
    from ..tasks.backup import restore_test_task
    restore_test_task.delay(dump_key=key)
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/offsite/restore", methods=["POST"])
async def restore_from_offsite():
    """Replace this install's database with a dump from the bucket, then
    fetch the library's files back. Body: {key, confirm}, where confirm is
    the dump's token from /offsite/dumps. Refused while a backup or another
    restore is going."""
    body = await request.get_json(silent=True) or {}
    key = body.get("key")
    if not isinstance(key, str) or not key.startswith("database/"):
        return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
    ts = key.removeprefix("database/fabledcurator-").removesuffix(".dump")
    expected = restore_token(ts)
    if body.get("confirm") != expected:
        return _bad("confirm_mismatch", detail=f"confirm must equal {expected!r}",
                    expected=expected)
    _, err = await _saved_target()
    if err:
        return err
    busy = await _busy_reason()
    if busy:
        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
    from ..tasks.backup import restore_offsite_task
    restore_offsite_task.delay(dump_key=key)
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/offsite/fetch-files", methods=["POST"])
async def fetch_missing_files():
    """Fetch every library file that is missing from disk back from the
    bucket, leaving the database as it is."""
    _, err = await _saved_target()
    if err:
        return err
    busy = await _busy_reason()
    if busy:
        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
    from ..tasks.backup import restore_offsite_task
    restore_offsite_task.delay(files_only=True)
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/runs", methods=["GET"])
async def list_runs():
    try:
        limit = min(int(request.args.get("limit", "50")), 200)
    except ValueError:
        return _bad("invalid_limit")
    if limit < 1:
        return _bad("invalid_limit")
    kind = request.args.get("kind")
    if kind is not None and kind not in _KINDS:
        return _bad("invalid_kind", detail=f"kind must be one of {sorted(_KINDS)}")
    before_id_raw = request.args.get("before_id")
    before_id = int(before_id_raw) if before_id_raw else None

    async with get_session() as session:
        stmt = select(BackupRun).order_by(desc(BackupRun.id))
        if kind:
            stmt = stmt.where(BackupRun.kind == kind)
        if before_id is not None:
            stmt = stmt.where(BackupRun.id < before_id)
        stmt = stmt.limit(limit + 1)
        rows = (await session.execute(stmt)).scalars().all()

    has_more = len(rows) > limit
    rows = rows[:limit]
    return jsonify({
        "runs": [_row_to_dict(r) for r in rows],
        "next_cursor": rows[-1].id if has_more and rows else None,
    })


@system_backup_bp.route("/runs/<int:run_id>", methods=["GET"])
async def get_run(run_id: int):
    async with get_session() as session:
        row = await session.get(BackupRun, run_id)
    if row is None:
        return _bad("not_found", status=404)
    return jsonify(_row_to_dict(row))


@system_backup_bp.route("/runs/<int:run_id>", methods=["PATCH"])
async def patch_run(run_id: int):
    body = await request.get_json(silent=True) or {}
    if "tag" not in body:
        return _bad("invalid_body", detail="tag required")
    tag = _validate_tag(body["tag"])
    if isinstance(tag, tuple):
        return tag

    async with get_session() as session:
        row = await session.get(BackupRun, run_id)
        if row is None:
            return _bad("not_found", status=404)
        row.tag = tag
        await session.commit()
        await session.refresh(row)
    return jsonify(_row_to_dict(row))


@system_backup_bp.route("/runs/<int:run_id>/restore", methods=["POST"])
async def trigger_restore(run_id: int):
    body = await request.get_json(silent=True) or {}
    supplied = body.get("confirm", "")

    async with get_session() as session:
        row = await session.get(BackupRun, run_id)
        if row is None:
            return _bad("not_found", status=404)
        if (row.manifest or {}).get("offsite"):
            return _bad(
                "not_restorable",
                detail=("this dump is kept in the off-site bucket, not on this server, "
                        "so it can't be restored from here"),
            )
        if row.kind != "db" or row.status != "ok" or not row.sql_path:
            return _bad(
                "not_restorable",
                detail="only a finished database dump kept on this server can be restored here",
            )
        expected = f"restore-{row.kind}-{row.id}"
        if supplied != expected:
            return _bad(
                "confirm_mismatch",
                detail=f"confirm must equal {expected!r}",
                expected=expected,
            )

    from ..tasks.backup import restore_db_task
    restore_db_task.delay(source_backup_run_id=run_id)
    return jsonify({"status": "dispatched", "kind": "db"}), 202


@system_backup_bp.route("/runs/<int:run_id>/test", methods=["POST"])
async def restore_test_local(run_id: int):
    """Test a database dump kept on this server, the same way as one in the
    bucket. Needs no confirmation."""
    async with get_session() as session:
        row = await session.get(BackupRun, run_id)
        if row is None:
            return _bad("not_found", status=404)
        if row.kind != "db" or row.status != "ok" or not row.sql_path:
            return _bad("not_testable",
                        detail="only a finished database dump on this server can be tested here")
    busy = await _restore_test_blocked()
    if busy:
        return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
    from ..tasks.backup import restore_test_task
    restore_test_task.delay(source_backup_run_id=run_id)
    return jsonify({"status": "dispatched"}), 202


@system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
async def delete_run(run_id: int):
    body = await request.get_json(silent=True) or {}
    supplied = body.get("confirm", "")

    async with get_session() as session:
        row = await session.get(BackupRun, run_id)
        if row is None:
            return _bad("not_found", status=404)
        expected = f"delete-{row.kind}-{row.id}"
        if supplied != expected:
            return _bad(
                "confirm_mismatch",
                detail=f"confirm must equal {expected!r}",
                expected=expected,
            )
        from ..services import backup_service
        backup_service.unlink_artifact_files(
            sql_path=row.sql_path,
            manifest_path=(row.manifest or {}).get("manifest_path"),
        )
        await session.delete(row)
        await session.commit()
    return "", 204


@system_backup_bp.route("/settings", methods=["GET"])
async def get_settings():
    async with get_session() as session:
        row = await ImportSettings.load(session)
    return jsonify({
        "backup_db_nightly_enabled": row.backup_db_nightly_enabled,
        "backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
        "backup_db_keep_last_n": row.backup_db_keep_last_n,
    })


@system_backup_bp.route("/destination", methods=["GET"])
async def get_destination():
    """The off-site destination as the settings screen shows it. The secret
    key is never returned, only whether one is saved."""
    from ..services import backup_destination as dest

    async with get_session() as session:
        row = await ImportSettings.load(session)
        return jsonify(dest.describe(row))


@system_backup_bp.route("/destination", methods=["PATCH"])
async def patch_destination():
    """Save the destination. Body {endpoint, bucket, access_key, region,
    secret_key, upload_limit_mbps}; an empty field clears it, and a missing
    upload_limit_mbps keeps the saved limit. A missing or empty secret_key
    keeps the saved one, so the screen can save the other fields without the
    secret being typed again."""
    from ..services import backup_destination as dest

    body = await request.get_json(silent=True)
    if not isinstance(body, dict):
        return _bad("invalid_body", detail="body must be a JSON object")
    fields, why = dest.clean_fields(body)
    if fields is None:
        return _bad("invalid_destination", detail=why)
    secret = body.get("secret_key")
    if secret is not None and not isinstance(secret, str):
        return _bad("invalid_destination", detail="secret_key must be text")
    limit = body.get("upload_limit_mbps")
    if limit is not None and (
        not isinstance(limit, int) or isinstance(limit, bool) or not 0 <= limit <= 100_000
    ):
        return _bad("invalid_destination",
                    detail="upload_limit_mbps must be a whole number from 0 (no limit) up")

    async with get_session() as session:
        row = await ImportSettings.load(session)
        if limit is not None:
            row.backup_upload_limit_mbps = limit
        row.backup_s3_endpoint = fields.endpoint
        row.backup_s3_bucket = fields.bucket
        row.backup_s3_access_key = fields.access_key
        row.backup_s3_region = fields.region
        if secret and secret.strip():
            row.backup_s3_secret_sealed = dest.seal(secret.strip())
        await session.commit()
        return jsonify(dest.describe(row))


@system_backup_bp.route("/destination", methods=["DELETE"])
async def delete_destination():
    """Forget the destination, secret included. What was already sent stays in
    the bucket: FabledCurator never deletes there."""
    from ..services import backup_destination as dest

    async with get_session() as session:
        row = await ImportSettings.load(session)
        row.backup_s3_endpoint = None
        row.backup_s3_bucket = None
        row.backup_s3_access_key = None
        row.backup_s3_secret_sealed = None
        row.backup_s3_region = dest.DEFAULT_REGION
        await session.commit()
        return jsonify(dest.describe(row))


@system_backup_bp.route("/destination/test", methods=["POST"])
async def try_destination():
    """Reach the bucket with the fields in the body (the form as typed, saved
    or not); a missing secret_key uses the saved one. Lists the bucket and
    writes nothing. Answers {ok, message} either way: a destination that
    can't be reached is a result to show, not an API error."""
    import asyncio

    from ..services import backup_destination as dest
    from ..services.backup_s3 import S3Error
    from ..services.credential_crypto import InvalidCredentialBlob

    body = await request.get_json(silent=True) or {}
    if not isinstance(body, dict):
        return _bad("invalid_body", detail="body must be a JSON object")
    fields, why = dest.clean_fields(body)
    if fields is None:
        return jsonify({"ok": False, "message": f"Not tested: {why}."})
    secret = body.get("secret_key")
    secret = secret.strip() if isinstance(secret, str) else ""
    if not secret:
        async with get_session() as session:
            row = await ImportSettings.load(session)
            sealed = row.backup_s3_secret_sealed
        if sealed:
            try:
                secret = dest.unseal(sealed)
            except InvalidCredentialBlob:
                return jsonify({"ok": False, "message": (
                    "Not tested: the saved secret key can't be opened on this server. "
                    "Enter the secret key again."
                )})
    target = dest.target_for(fields, secret or None)
    if target is None:
        missing = [name for name, v in (
            ("address", fields.endpoint), ("bucket", fields.bucket),
            ("access key", fields.access_key), ("secret key", secret),
        ) if not v]
        return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
    try:
        return jsonify(await asyncio.to_thread(dest.check_target, target))
    except S3Error as exc:
        return jsonify({"ok": False, "message": str(exc)})


@system_backup_bp.route("/settings", methods=["PATCH"])
async def patch_settings():
    body = await request.get_json(silent=True)
    if not isinstance(body, dict):
        return _bad("invalid_body", detail="body must be a JSON object")

    err = _validate_backup_settings_patch(body)
    if err is not None:
        return err

    async with get_session() as session:
        row = await ImportSettings.load(session)
        for field in _BACKUP_SETTINGS_FIELDS:
            if field in body:
                setattr(row, field, body[field])
        await session.commit()
    return await get_settings()