Merge pull request 'perf(migration): 0045 streams json_each (no materialize / no temp blowup)' (#94) from dev into main

alembic/versions/0045_image_prediction_table.py

@@ -55,25 +55,24 @@ def upgrade() -> None:
     # 0.70) right here so this is self-sufficient — it does NOT depend on the
     # #764 prune having run, and extracting only the >=floor tail keeps
     # image_prediction small (~tens of rows/image) even from the full JSON.
-    # MATERIALIZED so the json_typeof guard runs BEFORE json_each — some rows
+    # Guard json_each against non-object rows (some tagger_predictions are JSON
-    # store tagger_predictions as a JSON scalar/null (not an object), and
+    # scalars/null → "cannot deconstruct a scalar"). The inline CASE passes an
-    # json_each throws "cannot deconstruct a scalar" on those. Filtering to
+    # empty object for those, so json_each yields nothing — a single STREAMING
-    # objects in a materialized CTE keeps json_each from ever seeing them.
+    # pass with NO materialization/temp spill (an earlier MATERIALIZED-CTE guard
+    # forced ~100 GB to temp on NFS and was pathologically slow).
     op.execute(
         """
-        WITH objs AS MATERIALIZED (
-            SELECT id, tagger_predictions AS preds
-            FROM image_record
-            WHERE tagger_predictions IS NOT NULL
-              AND json_typeof(tagger_predictions) = 'object'
-        )
         INSERT INTO image_prediction (image_record_id, raw_name, category, score)
-        SELECT o.id,
+        SELECT ir.id,
                je.key,
                COALESCE(je.value ->> 'category', 'general'),
                (je.value ->> 'confidence')::double precision
-        FROM objs o,
+        FROM image_record ir,
-             json_each(o.preds) je
+             json_each(
+                 CASE WHEN json_typeof(ir.tagger_predictions) = 'object'
+                      THEN ir.tagger_predictions
+                      ELSE '{}'::json END
+             ) je
         WHERE je.value ->> 'confidence' IS NOT NULL
           AND (je.value ->> 'confidence')::double precision
               >= COALESCE(