Schema reconciliation + index hygiene, and the weekly base-image refresh #243
@@ -0,0 +1,178 @@
|
||||
|
||||
# TEMPORARY — milestone 328 steps 1-2. Delete once the baseline is stamped.
|
||||
#
|
||||
# Squashing 87 alembic revisions into one baseline has exactly one dangerous
|
||||
# failure: the generated baseline does not reproduce the schema the chain
|
||||
# produced, `alembic stamp` writes a version string anyway (it validates
|
||||
# NOTHING), and the divergence surfaces on the next real migration against the
|
||||
# operator's live data.
|
||||
#
|
||||
# So this workflow does the comparison in CI, where a pgvector Postgres already
|
||||
# gets built from the chain on every integration run, and nothing is at risk.
|
||||
# It answers one question: does `upgrade head` on the collapsed chain produce a
|
||||
# byte-identical schema to `upgrade head` on the 87-revision chain?
|
||||
#
|
||||
# The chain is read from git rather than from the working tree, so this keeps
|
||||
# working AFTER the old revisions are deleted — `chain_ref` names a commit that
|
||||
# still has them. That is what makes this the proof for step 1 and the
|
||||
# pre-flight for step 2, rather than a one-shot script.
|
||||
#
|
||||
# `generate: true` additionally autogenerates a candidate baseline from the
|
||||
# models and uploads it. That is a starting point, NOT the answer: autogenerate
|
||||
# reads SQLAlchemy metadata, and three things in this schema do not live there —
|
||||
# * CREATE EXTENSION vector (0001)
|
||||
# * CREATE EXTENSION tsm_system_rows (0004)
|
||||
# * the HNSW index on image_record.siglip_embedding, which is raw SQL
|
||||
# because alembic's create_index cannot express `USING hnsw (...)` (0036)
|
||||
# plus any CHECK constraint or server_default that a migration added without
|
||||
# the model declaring it. Those must be hand-added, and the diff below is what
|
||||
# proves none were missed.
|
||||
name: Alembic baseline
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
chain_ref:
|
||||
description: 'Commit/tag that still carries the full 0001..0087 chain'
|
||||
type: string
|
||||
default: '0a5bbe8'
|
||||
generate:
|
||||
description: 'Also autogenerate a candidate baseline from the models'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
compare:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Full history is the point: `chain_ref` is read out of git, so a
|
||||
# shallow clone would not have the revisions to compare against.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve the Postgres service and install deps
|
||||
run: |
|
||||
set -eux
|
||||
# Same service-IP dance as ci.yml's integration job; see the long
|
||||
# comment there for why the job name must stay separator-free.
|
||||
PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
test -n "$PG"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
test -n "$PG_IP"
|
||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||
echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt
|
||||
else
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
# DB 1: the 87-revision chain, read out of git at `chain_ref`.
|
||||
#
|
||||
# A git worktree rather than a checkout, so the current tree — which is
|
||||
# what we are testing — is left completely alone.
|
||||
- name: Build the schema the OLD chain produces
|
||||
env:
|
||||
CHAIN_REF: ${{ github.event.inputs.chain_ref }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain
|
||||
git worktree add /tmp/chain "$CHAIN_REF"
|
||||
ls /tmp/chain/alembic/versions/*.py | wc -l
|
||||
cd /tmp/chain
|
||||
DB_NAME=fc_chain alembic upgrade head
|
||||
cd -
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_chain > chain.sql
|
||||
wc -l chain.sql
|
||||
|
||||
# Optional: a candidate baseline, autogenerated from the models against an
|
||||
# EMPTY database so every table shows up as a create. Uploaded for a human
|
||||
# to finish — it will be missing the three raw-SQL items named at the top.
|
||||
- name: Autogenerate a candidate baseline
|
||||
if: ${{ github.event.inputs.generate == 'true' }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_gen
|
||||
# Hide the existing revisions so alembic sees an empty history and
|
||||
# emits the whole schema rather than a delta.
|
||||
mkdir -p /tmp/versions_held
|
||||
mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true
|
||||
DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true
|
||||
# Printed in full rather than uploaded. ci-requirements.md records
|
||||
# that this runner cannot do actions/upload-artifact@v4+, and the
|
||||
# repo dropped the action entirely in 2026-05; the job log is the
|
||||
# retrieval channel that is actually proven here.
|
||||
echo "===== BEGIN CANDIDATE BASELINE ====="
|
||||
cat alembic/versions/*.py
|
||||
echo "===== END CANDIDATE BASELINE ====="
|
||||
# Put the tree back exactly as it was; this job never mutates state.
|
||||
rm -f alembic/versions/*.py
|
||||
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
||||
|
||||
# DB 2: whatever the CURRENT tree's alembic/versions produces. Before the
|
||||
# squash that is the same 87 revisions and the diff is trivially clean —
|
||||
# which is worth running once as a control, so a clean diff after the
|
||||
# squash means something.
|
||||
- name: Build the schema the CURRENT tree produces
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
||||
ls alembic/versions/*.py | wc -l
|
||||
DB_NAME=fc_base alembic upgrade head
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_base > baseline.sql
|
||||
wc -l baseline.sql
|
||||
|
||||
# The verdict.
|
||||
#
|
||||
# pg_dump orders dumpable objects by name within type, not by creation
|
||||
# order, so two schemas built by different routes are directly
|
||||
# comparable. The only normalisation applied is dropping blank lines,
|
||||
# comment lines and the alembic_version row-count noise — deliberately
|
||||
# minimal, because a filter that hides a real difference is the one way
|
||||
# this check passes when it should fail. Whatever is normalised is
|
||||
# printed, so the filtering itself is reviewable.
|
||||
- name: Diff
|
||||
run: |
|
||||
set -eu
|
||||
norm() {
|
||||
grep -vE '^\s*(--|$)' "$1" | sed 's/[[:space:]]*$//'
|
||||
}
|
||||
norm chain.sql > a.txt
|
||||
norm baseline.sql > b.txt
|
||||
echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines"
|
||||
if diff -u a.txt b.txt > schema.diff; then
|
||||
echo "SCHEMAS IDENTICAL — the collapsed chain reproduces the old one."
|
||||
else
|
||||
echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' schema.diff) changed lines:"
|
||||
cat schema.diff
|
||||
echo
|
||||
echo "The baseline is wrong, not the database. Do not stamp."
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user