From e6d5f67f11847a7c5f11a0dee8bed95b0a1a278c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 10 Jun 2026 20:58:47 -0400 Subject: [PATCH] perf(migration): 0045 streams json_each via inline CASE guard (no temp spill) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MATERIALIZED-CTE scalar guard forced Postgres to materialize all object rows with their full JSON (~100 GB) to temp before json_each — on NFS that's a huge spill and pathologically slow (risks disk-full). Replace with an inline CASE that feeds json_each an empty object for non-object rows: same scalar guard, but a single streaming pass with no materialization. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../versions/0045_image_prediction_table.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/alembic/versions/0045_image_prediction_table.py b/alembic/versions/0045_image_prediction_table.py index de78507..ebfa6c2 100644 --- a/alembic/versions/0045_image_prediction_table.py +++ b/alembic/versions/0045_image_prediction_table.py @@ -55,25 +55,24 @@ def upgrade() -> None: # 0.70) right here so this is self-sufficient — it does NOT depend on the # #764 prune having run, and extracting only the >=floor tail keeps # image_prediction small (~tens of rows/image) even from the full JSON. - # MATERIALIZED so the json_typeof guard runs BEFORE json_each — some rows - # store tagger_predictions as a JSON scalar/null (not an object), and - # json_each throws "cannot deconstruct a scalar" on those. Filtering to - # objects in a materialized CTE keeps json_each from ever seeing them. + # Guard json_each against non-object rows (some tagger_predictions are JSON + # scalars/null → "cannot deconstruct a scalar"). The inline CASE passes an + # empty object for those, so json_each yields nothing — a single STREAMING + # pass with NO materialization/temp spill (an earlier MATERIALIZED-CTE guard + # forced ~100 GB to temp on NFS and was pathologically slow). op.execute( """ - WITH objs AS MATERIALIZED ( - SELECT id, tagger_predictions AS preds - FROM image_record - WHERE tagger_predictions IS NOT NULL - AND json_typeof(tagger_predictions) = 'object' - ) INSERT INTO image_prediction (image_record_id, raw_name, category, score) - SELECT o.id, + SELECT ir.id, je.key, COALESCE(je.value ->> 'category', 'general'), (je.value ->> 'confidence')::double precision - FROM objs o, - json_each(o.preds) je + FROM image_record ir, + json_each( + CASE WHEN json_typeof(ir.tagger_predictions) = 'object' + THEN ir.tagger_predictions + ELSE '{}'::json END + ) je WHERE je.value ->> 'confidence' IS NOT NULL AND (je.value ->> 'confidence')::double precision >= COALESCE(