From ed2b1adc2e53a542baa69cb05bec6ee227b43bf1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:43:26 -0400 Subject: [PATCH] ci: compare the schema the MODELS produce against the migrations (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit baseline.yml only ever compared migrations against migrations. The question #3275 exists because nobody had ever asked the other one: does a database built from the MODELS match the one the chain produces? `mode: models` answers it. It applies the candidate autogenerated from the models instead of this tree's revisions, and diffs that against the chain. A clean run means --autogenerate is trustworthy again, which it demonstrably has not been: against the pre-reconciliation models it would have proposed dropping eleven indexes and two uniqueness guarantees. The two extensions are created by hand in that mode. They are database objects rather than table metadata, so no model can carry them — their absence is outside what this comparison asks about, and silently tolerating it is correct rather than a filter that hides a defect. Also declares the HNSW index on the ImageRecord model. SQLAlchemy can express an hnsw access method with an operator class (postgresql_using + postgresql_ops), so there was never a reason for it to live only in 0036. That removes the last item from the list of things a generated baseline cannot reproduce, leaving only the two extensions. --- .forgejo/workflows/baseline.yml | 40 +++++++++++++++++++++++++----- backend/app/models/image_record.py | 11 ++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index e7df685..14643f8 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -37,6 +37,10 @@ on: description: 'Commit/tag that still carries the full 0001..0087 chain' type: string default: '0a5bbe8' + mode: + description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS' + type: string + default: 'chain' jobs: compare: @@ -174,20 +178,44 @@ jobs: echo "candidate-bytes: $(wc -c < "$F")" echo "candidate-b64-lines: $(echo "$B64" | wc -l)" set -x + mkdir -p /tmp/candidate + cp alembic/versions/*.py /tmp/candidate/ # Put the tree back exactly as it was; this job never mutates state. rm -f alembic/versions/*.py mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true - # DB 2: whatever the CURRENT tree's alembic/versions produces. Before the - # squash that is the same 87 revisions and the diff is trivially clean — - # which is worth running once as a control, so a clean diff after the - # squash means something. + # DB 2: what the CURRENT tree produces. + # + # `mode: models` applies the candidate autogenerated from the MODELS + # instead, which is what answers "do the models describe the schema?" — + # the question #3275 exists because nobody had ever asked it. Under that + # mode a clean diff means autogenerate is trustworthy again. + # + # The two extensions are created by hand first. They are database + # objects, not table metadata, so no model can carry them and their + # absence is not a model defect — it is simply outside what this + # comparison is asking about. - name: Build the schema the CURRENT tree produces + env: + MODE: ${{ github.event.inputs.mode }} run: | set -eux docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base - ls alembic/versions/*.py | wc -l - DB_NAME=fc_base alembic upgrade head + if [ "${MODE:-chain}" = "models" ]; then + docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \ + -c "CREATE EXTENSION IF NOT EXISTS vector" \ + -c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows" + mkdir -p /tmp/held + mv alembic/versions/*.py /tmp/held/ + cp /tmp/candidate/*.py alembic/versions/ + ls alembic/versions/*.py + DB_NAME=fc_base alembic upgrade head + rm -f alembic/versions/*.py + mv /tmp/held/*.py alembic/versions/ + else + ls alembic/versions/*.py | wc -l + DB_NAME=fc_base alembic upgrade head + fi docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \ --no-owner --no-privileges -d fc_base > baseline.sql wc -l baseline.sql diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index be273ce..3de0a3f 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -33,6 +33,17 @@ class ImageRecord(Base): __table_args__ = ( + # alembic 0036, and the last thing in this schema that lived only in a + # migration. SQLAlchemy CAN express an hnsw index with an operator + # class, so there is no reason for it to be invisible to the models — + # and its absence was the quietest failure of the lot: everything + # works, similarity search just silently stops using an index. + Index( + "ix_image_record_siglip_hnsw", + "siglip_embedding", + postgresql_using="hnsw", + postgresql_ops={"siglip_embedding": "vector_cosine_ops"}, + ), # alembic 0035/0071: the date-ordered browse indexes (#3275). Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")), Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")),