fix(importer): catch PIL OSError during transparency + phash blocks, skip as invalid_image instead of letting Celery autoretry loop forever

Operator hit a corrupt JPEG in the IR set 2026-05-25: PIL.verify() only
validates header structure but doesn't catch truncated/broken pixel
data. The error surfaces later in _transparency_pct (via getchannel
'A' -> load) or compute_phash (load) — both blow up with OSError
'broken data stream when reading image file'. Celery's autoretry_for
then bounces the same file forever instead of marking it skipped.

Wrap both PIL.load-triggering call sites with try/except OSError ->
ImportResult(status=skipped, skip_reason=invalid_image).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-25 02:34:36 -04:00
parent 52445eb501
commit 68cffce322
2 changed files with 77 additions and 3 deletions
+22 -3
View File
@@ -280,7 +280,18 @@ class Importer:
)
if self.settings.skip_transparent and has_alpha:
pct = self._transparency_pct(source)
try:
pct = self._transparency_pct(source)
except OSError as exc:
# PIL.verify() at line 263 only validates header structure;
# truncated/corrupt pixel data only surfaces when load()
# actually decodes (here via getchannel('A')). Convert to
# invalid_image skip so the Celery autoretry loop doesn't
# bounce the same broken file forever.
return ImportResult(
status="skipped", skip_reason=SkipReason.invalid_image,
error=f"PIL load failed during transparency check: {exc}",
)
if pct >= self.settings.transparency_threshold:
return ImportResult(
status="skipped", skip_reason=SkipReason.too_transparent,
@@ -302,8 +313,16 @@ class Importer:
# Perceptual near-dup (images only; videos keep phash NULL).
phash = None
if not is_video(source):
with Image.open(source) as im:
phash = compute_phash(im)
try:
with Image.open(source) as im:
phash = compute_phash(im)
except OSError as exc:
# Same rationale as the transparency-check guard above:
# broken-pixel-data files pass verify() but blow up here.
return ImportResult(
status="skipped", skip_reason=SkipReason.invalid_image,
error=f"PIL load failed during phash compute: {exc}",
)
if phash is not None:
cand_rows = self.session.execute(
select(
+55
View File
@@ -134,3 +134,58 @@ def test_root_level_file_has_no_artist(importer, import_layout):
importer.import_one(src)
artists = importer.session.execute(select(Artist)).scalars().all()
assert artists == []
def test_pil_load_oserror_in_transparency_check_skips_not_raises(
importer, import_layout, monkeypatch,
):
"""PIL.verify() only validates header structure — broken pixel data
only surfaces when load() actually decodes. The importer must catch
the OSError and return a skipped: invalid_image result so the Celery
autoretry loop doesn't bounce the same broken file forever.
Operator hit this 2026-05-25 with a corrupt JPEG in the IR set."""
import_root, _ = import_layout
src = import_root / "Bob" / "corrupt.png"
# Make a real RGBA PNG so the has_alpha path engages.
_make_png_rgba(src, (100, 100), alpha=128)
importer.settings.skip_transparent = True
importer.settings.transparency_threshold = 0.5
# Force the next _transparency_pct call to raise as if PIL's load()
# blew up on truncated pixel data.
def _boom(_self, _src):
raise OSError("broken data stream when reading image file")
monkeypatch.setattr(
type(importer), "_transparency_pct", _boom,
)
result = importer.import_one(src)
assert result.status == "skipped"
assert result.skip_reason == SkipReason.invalid_image
assert "transparency check" in (result.error or "")
def test_pil_load_oserror_in_phash_compute_skips_not_raises(
importer, import_layout, monkeypatch,
):
"""Same shape as the transparency-check guard, but for the phash
compute block — the OTHER place PIL.load() runs implicitly during
the dedup pipeline."""
import_root, _ = import_layout
src = import_root / "Carol" / "corrupt.jpg"
_make_jpeg(src)
# Disable transparency check so we reach the phash compute block.
importer.settings.skip_transparent = False
from backend.app.services import importer as importer_module
def _boom(_im):
raise OSError("broken data stream when reading image file")
monkeypatch.setattr(importer_module, "compute_phash", _boom)
result = importer.import_one(src)
assert result.status == "skipped"
assert result.skip_reason == SkipReason.invalid_image
assert "phash compute" in (result.error or "")