bvandeusen
e6d5f67f11
The MATERIALIZED-CTE scalar guard forced Postgres to materialize all object rows with their full JSON (~100 GB) to temp before json_each — on NFS that's a huge spill and pathologically slow (risks disk-full). Replace with an inline CASE that feeds json_each an empty object for non-object rows: same scalar guard, but a single streaming pass with no materialization. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
91 lines
3.6 KiB
Python
91 lines
3.6 KiB
Python
"""image_prediction table + backfill from image_record.tagger_predictions
|
|
|
|
Normalizes the per-image tagger predictions out of the JSON blob into a
|
|
queryable table (#768). Backfills from the existing JSON in one set-based
|
|
INSERT…SELECT over json_each — fast because the #764 prune already shrank
|
|
each row to its >=0.70 entries. The old image_record.tagger_predictions
|
|
column is left in place here (vestigial) and dropped in a follow-up once the
|
|
code cutover is verified — dropping it needs an ACCESS EXCLUSIVE lock on the
|
|
hot image_record table (the 0044 lock class), so it's deferred to a
|
|
quiesced-worker window.
|
|
|
|
Revision ID: 0045
|
|
Revises: 0044
|
|
Create Date: 2026-06-10
|
|
|
|
"""
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
revision: str = "0045"
|
|
down_revision: Union[str, None] = "0044"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"image_prediction",
|
|
sa.Column("id", sa.Integer(), primary_key=True),
|
|
sa.Column(
|
|
"image_record_id", sa.Integer(),
|
|
sa.ForeignKey("image_record.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("raw_name", sa.String(length=255), nullable=False),
|
|
sa.Column("category", sa.String(length=64), nullable=False),
|
|
sa.Column("score", sa.Float(), nullable=False),
|
|
sa.UniqueConstraint(
|
|
"image_record_id", "raw_name", name="image_raw_name",
|
|
),
|
|
)
|
|
op.create_index(
|
|
"ix_image_prediction_image", "image_prediction", ["image_record_id"],
|
|
)
|
|
op.create_index(
|
|
"ix_image_prediction_name_score", "image_prediction",
|
|
["raw_name", "score"],
|
|
)
|
|
# Backfill from the JSON blob. json_each expands {name: {category,
|
|
# confidence}} into one row per prediction. category defaults to 'general'
|
|
# to mirror the suggestion read path; rows with no confidence are skipped.
|
|
# Filter to >= the store floor (ml_settings.tagger_store_floor, default
|
|
# 0.70) right here so this is self-sufficient — it does NOT depend on the
|
|
# #764 prune having run, and extracting only the >=floor tail keeps
|
|
# image_prediction small (~tens of rows/image) even from the full JSON.
|
|
# Guard json_each against non-object rows (some tagger_predictions are JSON
|
|
# scalars/null → "cannot deconstruct a scalar"). The inline CASE passes an
|
|
# empty object for those, so json_each yields nothing — a single STREAMING
|
|
# pass with NO materialization/temp spill (an earlier MATERIALIZED-CTE guard
|
|
# forced ~100 GB to temp on NFS and was pathologically slow).
|
|
op.execute(
|
|
"""
|
|
INSERT INTO image_prediction (image_record_id, raw_name, category, score)
|
|
SELECT ir.id,
|
|
je.key,
|
|
COALESCE(je.value ->> 'category', 'general'),
|
|
(je.value ->> 'confidence')::double precision
|
|
FROM image_record ir,
|
|
json_each(
|
|
CASE WHEN json_typeof(ir.tagger_predictions) = 'object'
|
|
THEN ir.tagger_predictions
|
|
ELSE '{}'::json END
|
|
) je
|
|
WHERE je.value ->> 'confidence' IS NOT NULL
|
|
AND (je.value ->> 'confidence')::double precision
|
|
>= COALESCE(
|
|
(SELECT tagger_store_floor FROM ml_settings WHERE id = 1),
|
|
0.70
|
|
)
|
|
ON CONFLICT (image_record_id, raw_name) DO NOTHING
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_index("ix_image_prediction_name_score", "image_prediction")
|
|
op.drop_index("ix_image_prediction_image", "image_prediction")
|
|
op.drop_table("image_prediction")
|