Schema reconciliation + index hygiene, and the weekly base-image refresh #243
@@ -37,6 +37,10 @@ on:
|
|||||||
description: 'Commit/tag that still carries the full 0001..0087 chain'
|
description: 'Commit/tag that still carries the full 0001..0087 chain'
|
||||||
type: string
|
type: string
|
||||||
default: '0a5bbe8'
|
default: '0a5bbe8'
|
||||||
|
mode:
|
||||||
|
description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS'
|
||||||
|
type: string
|
||||||
|
default: 'chain'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
compare:
|
compare:
|
||||||
@@ -174,20 +178,44 @@ jobs:
|
|||||||
echo "candidate-bytes: $(wc -c < "$F")"
|
echo "candidate-bytes: $(wc -c < "$F")"
|
||||||
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
||||||
set -x
|
set -x
|
||||||
|
mkdir -p /tmp/candidate
|
||||||
|
cp alembic/versions/*.py /tmp/candidate/
|
||||||
# Put the tree back exactly as it was; this job never mutates state.
|
# Put the tree back exactly as it was; this job never mutates state.
|
||||||
rm -f alembic/versions/*.py
|
rm -f alembic/versions/*.py
|
||||||
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
||||||
|
|
||||||
# DB 2: whatever the CURRENT tree's alembic/versions produces. Before the
|
# DB 2: what the CURRENT tree produces.
|
||||||
# squash that is the same 87 revisions and the diff is trivially clean —
|
#
|
||||||
# which is worth running once as a control, so a clean diff after the
|
# `mode: models` applies the candidate autogenerated from the MODELS
|
||||||
# squash means something.
|
# instead, which is what answers "do the models describe the schema?" —
|
||||||
|
# the question #3275 exists because nobody had ever asked it. Under that
|
||||||
|
# mode a clean diff means autogenerate is trustworthy again.
|
||||||
|
#
|
||||||
|
# The two extensions are created by hand first. They are database
|
||||||
|
# objects, not table metadata, so no model can carry them and their
|
||||||
|
# absence is not a model defect — it is simply outside what this
|
||||||
|
# comparison is asking about.
|
||||||
- name: Build the schema the CURRENT tree produces
|
- name: Build the schema the CURRENT tree produces
|
||||||
|
env:
|
||||||
|
MODE: ${{ github.event.inputs.mode }}
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
||||||
ls alembic/versions/*.py | wc -l
|
if [ "${MODE:-chain}" = "models" ]; then
|
||||||
DB_NAME=fc_base alembic upgrade head
|
docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \
|
||||||
|
-c "CREATE EXTENSION IF NOT EXISTS vector" \
|
||||||
|
-c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows"
|
||||||
|
mkdir -p /tmp/held
|
||||||
|
mv alembic/versions/*.py /tmp/held/
|
||||||
|
cp /tmp/candidate/*.py alembic/versions/
|
||||||
|
ls alembic/versions/*.py
|
||||||
|
DB_NAME=fc_base alembic upgrade head
|
||||||
|
rm -f alembic/versions/*.py
|
||||||
|
mv /tmp/held/*.py alembic/versions/
|
||||||
|
else
|
||||||
|
ls alembic/versions/*.py | wc -l
|
||||||
|
DB_NAME=fc_base alembic upgrade head
|
||||||
|
fi
|
||||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||||
--no-owner --no-privileges -d fc_base > baseline.sql
|
--no-owner --no-privileges -d fc_base > baseline.sql
|
||||||
wc -l baseline.sql
|
wc -l baseline.sql
|
||||||
|
|||||||
@@ -33,6 +33,17 @@ class ImageRecord(Base):
|
|||||||
|
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# alembic 0036, and the last thing in this schema that lived only in a
|
||||||
|
# migration. SQLAlchemy CAN express an hnsw index with an operator
|
||||||
|
# class, so there is no reason for it to be invisible to the models —
|
||||||
|
# and its absence was the quietest failure of the lot: everything
|
||||||
|
# works, similarity search just silently stops using an index.
|
||||||
|
Index(
|
||||||
|
"ix_image_record_siglip_hnsw",
|
||||||
|
"siglip_embedding",
|
||||||
|
postgresql_using="hnsw",
|
||||||
|
postgresql_ops={"siglip_embedding": "vector_cosine_ops"},
|
||||||
|
),
|
||||||
# alembic 0035/0071: the date-ordered browse indexes (#3275).
|
# alembic 0035/0071: the date-ordered browse indexes (#3275).
|
||||||
Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")),
|
Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")),
|
||||||
Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")),
|
Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")),
|
||||||
|
|||||||
Reference in New Issue
Block a user