From 75eab188c8cfa374d749b05ecf1cc8fb313de7ca Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 10 Jun 2026 19:37:38 -0400 Subject: [PATCH] fix(migration): 0045 backfill filters to >= store floor (supersedes #764 prune) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #764 in-place prune (rewrite tagger_predictions to >=0.70) is too slow on 100 GB of TOAST and fails at its soft limit (interrupts a query mid-flight -> 'another command is already in progress'). #768 supersedes it: extract only the >=floor predictions into image_prediction via this set-based backfill, then drop the column (step 3) — reading 100 GB once + writing ~840k small rows beats rewriting 100 GB in place. So this backfill no longer assumes the prune ran: it filters by ml_settings.tagger_store_floor (default 0.70) itself, handling the full or partially-pruned JSON identically. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0045_image_prediction_table.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/alembic/versions/0045_image_prediction_table.py b/alembic/versions/0045_image_prediction_table.py index b3bec55..189fb1a 100644 --- a/alembic/versions/0045_image_prediction_table.py +++ b/alembic/versions/0045_image_prediction_table.py @@ -51,6 +51,10 @@ def upgrade() -> None: # Backfill from the JSON blob. json_each expands {name: {category, # confidence}} into one row per prediction. category defaults to 'general' # to mirror the suggestion read path; rows with no confidence are skipped. + # Filter to >= the store floor (ml_settings.tagger_store_floor, default + # 0.70) right here so this is self-sufficient — it does NOT depend on the + # #764 prune having run, and extracting only the >=floor tail keeps + # image_prediction small (~tens of rows/image) even from the full JSON. op.execute( """ INSERT INTO image_prediction (image_record_id, raw_name, category, score) @@ -62,6 +66,11 @@ def upgrade() -> None: json_each(ir.tagger_predictions) je WHERE ir.tagger_predictions IS NOT NULL AND je.value ->> 'confidence' IS NOT NULL + AND (je.value ->> 'confidence')::double precision + >= COALESCE( + (SELECT tagger_store_floor FROM ml_settings WHERE id = 1), + 0.70 + ) ON CONFLICT (image_record_id, raw_name) DO NOTHING """ )