diff --git a/alembic/versions/0045_image_prediction_table.py b/alembic/versions/0045_image_prediction_table.py index de78507..ebfa6c2 100644 --- a/alembic/versions/0045_image_prediction_table.py +++ b/alembic/versions/0045_image_prediction_table.py @@ -55,25 +55,24 @@ def upgrade() -> None: # 0.70) right here so this is self-sufficient — it does NOT depend on the # #764 prune having run, and extracting only the >=floor tail keeps # image_prediction small (~tens of rows/image) even from the full JSON. - # MATERIALIZED so the json_typeof guard runs BEFORE json_each — some rows - # store tagger_predictions as a JSON scalar/null (not an object), and - # json_each throws "cannot deconstruct a scalar" on those. Filtering to - # objects in a materialized CTE keeps json_each from ever seeing them. + # Guard json_each against non-object rows (some tagger_predictions are JSON + # scalars/null → "cannot deconstruct a scalar"). The inline CASE passes an + # empty object for those, so json_each yields nothing — a single STREAMING + # pass with NO materialization/temp spill (an earlier MATERIALIZED-CTE guard + # forced ~100 GB to temp on NFS and was pathologically slow). op.execute( """ - WITH objs AS MATERIALIZED ( - SELECT id, tagger_predictions AS preds - FROM image_record - WHERE tagger_predictions IS NOT NULL - AND json_typeof(tagger_predictions) = 'object' - ) INSERT INTO image_prediction (image_record_id, raw_name, category, score) - SELECT o.id, + SELECT ir.id, je.key, COALESCE(je.value ->> 'category', 'general'), (je.value ->> 'confidence')::double precision - FROM objs o, - json_each(o.preds) je + FROM image_record ir, + json_each( + CASE WHEN json_typeof(ir.tagger_predictions) = 'object' + THEN ir.tagger_predictions + ELSE '{}'::json END + ) je WHERE je.value ->> 'confidence' IS NOT NULL AND (je.value ->> 'confidence')::double precision >= COALESCE(