feat(backup): restore a same-version backup in place, history kept, all or nothing (#5815)

#5815 "Restore engine: in place for a same-version backup".

restore_in_place (steward/backup/restore.py) runs in one transaction:
1. It keeps aside what Steward records as it runs: ACTIVITY columns of rows
   that come back, and STATE_SETTINGS.
2. It drops every foreign key touching a replaced table.
3. It truncates the replaced tables and COPYs the backup's rows in
   unparsed, through asyncpg.
4. It puts the kept state back and moves sequences past their rows.
5. It puts right the rows whose parent didn't come back, by each key's own
   ON DELETE. A RESTRICT key that may be empty is cleared, so a run whose
   user is gone keeps its record.
6. It re-adds the keys, which checks every row.

A failure anywhere rolls the whole restore back. A backup made at another
schema is refused with NotSameSchema ("older": a full replace is the way;
"newer": update Steward).

The COPY-text reading moves into steward/backup/pgdata.py, shared by the
check and the restore. The integration tests stand in for pg_restore with
this database's own COPY output:
- settings come back while the audit log and the history of what remains
  stay;
- history of what didn't come back goes;
- a failure part-way changes nothing;
- a newer schema is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/backup/check.py

@@ -36,15 +36,14 @@ from sqlalchemy.ext.asyncio import create_async_engine
 from .contents import (ACTIVITY, ACTIVITY_EVERYWHERE, AREAS, EXCLUDED, HAND_SET_ONLY,
                        STATE_SETTINGS)
 from .dump import BackupError, read_backup
-from .restore import _async_url
+from .pgdata import async_url as _async_url
+from .pgdata import copy_rows, restore_script, script_tables
 
-CHECK_DEADLINE_SECONDS = 5 * 60
 # Names shown for a row, the first a table has; else its primary key.
 LABEL_COLUMNS = ("name", "username", "key", "title", "widget_key", "label", "mac")
 # A setting's two values are shown when they fit; longer ones are cut.
 SHOWN_CHARS = 120
 _TOKEN = re.compile(r"enc:v1:[A-Za-z0-9_\-=]+")
-_COPY = re.compile(r'^COPY ([^ ]+) \((.*)\) FROM stdin;$')
 
 
 @dataclass
@@ -134,79 +133,6 @@ class CheckReport:
         return {"ok": self.ok, "changes": self.changes, "rows": self.rows, "summary": self.summary()}
 
 
-# ── COPY text ───────────────────────────────────────────────────────────────
-
-_ESCAPES = {ord("b"): b"\b", ord("f"): b"\f", ord("n"): b"\n", ord("r"): b"\r",
-            ord("t"): b"\t", ord("v"): b"\v"}
-
-
-def unescape(field: bytes) -> str | None:
-    """One field of COPY's text format: \\N is NULL, backslash escapes are
-    undone (named, octal \\ooo and hex \\xhh)."""
-    if field == b"\\N":
-        return None
-    if b"\\" not in field:
-        return field.decode("utf-8", errors="replace")
-    out = bytearray()
-    i, n = 0, len(field)
-    while i < n:
-        c = field[i]
-        if c != 0x5C or i + 1 == n:
-            out.append(c)
-            i += 1
-            continue
-        nxt = field[i + 1]
-        if nxt in _ESCAPES:
-            out += _ESCAPES[nxt]
-            i += 2
-        elif 0x30 <= nxt <= 0x37:
-            j = i + 1
-            while j < n and j < i + 4 and 0x30 <= field[j] <= 0x37:
-                j += 1
-            out.append(int(field[i + 1:j], 8) & 0xFF)
-            i = j
-        elif nxt == ord("x") and i + 2 < n and chr(field[i + 2]) in "0123456789abcdefABCDEF":
-            j = i + 2
-            while j < n and j < i + 4 and chr(field[j]) in "0123456789abcdefABCDEF":
-                j += 1
-            out.append(int(field[i + 2:j], 16))
-            i = j
-        else:
-            out.append(nxt)
-            i += 2
-    return bytes(out).decode("utf-8", errors="replace")
-
-
-def copy_rows(data: bytes) -> list[list[str | None]]:
-    """COPY text rows (no header, no terminator) as lists of fields."""
-    return [[unescape(f) for f in line.split(b"\t")]
-            for line in data.split(b"\n") if line and line != b"\\."]
-
-
-def _ident(name: str) -> str:
-    name = name.strip()
-    return name[1:-1].replace('""', '"') if name.startswith('"') and name.endswith('"') else name
-
-
-def script_tables(script: bytes) -> dict[str, tuple[list[str], list[list[str | None]]]]:
-    """Each table's (columns, rows) from a pg_restore --data-only script."""
-    out: dict[str, tuple[list[str], list[list[str | None]]]] = {}
-    lines = iter(script.split(b"\n"))
-    for line in lines:
-        m = _COPY.match(line.decode("utf-8", errors="replace"))
-        if not m:
-            continue
-        table = _ident(m.group(1).split(".")[-1])
-        columns = [_ident(c) for c in m.group(2).split(",")]
-        body = []
-        for row in lines:
-            if row == b"\\.":
-                break
-            body.append(row)
-        out[table] = (columns, copy_rows(b"\n".join(body)))
-    return out
-
-
 # ── Comparing ───────────────────────────────────────────────────────────────
 
 def _fernet(key: str):
@@ -355,26 +281,8 @@ def _labels(tables: dict[str, _Side]) -> dict[str, dict[str, str]]:
 # ── Reading both sides ──────────────────────────────────────────────────────
 
 async def _backup_rows(dump: Path) -> dict[str, tuple[list[str], list[list[str | None]]]]:
-    """Every table's rows in the dump, as pg_restore would load them. Run
-    without a database: pg_restore writes the script and nothing connects."""
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            "pg_restore", "--data-only", "--no-owner", "--no-privileges", "--file=-", str(dump),
-            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
-    except FileNotFoundError:
-        raise BackupError("pg_restore isn't installed here.") from None
-    try:
-        out, err = await asyncio.wait_for(proc.communicate(), CHECK_DEADLINE_SECONDS)
-    except TimeoutError:
-        proc.kill()
-        await proc.wait()
-        raise BackupError(f"Reading the backup didn't finish within "
-                          f"{CHECK_DEADLINE_SECONDS // 60} minutes.") from None
-    if proc.returncode != 0:
-        lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()]
-        raise BackupError("The backup can't be read: " + (lines[-1] if lines else
-                                                           f"pg_restore exited {proc.returncode}."))
-    return script_tables(out)
+    """Every table's rows in the dump, as pg_restore would load them."""
+    return script_tables(await restore_script(dump))
 
 
 async def _here(database_url: str, tables: list[str]):