From d6d31843618a2379ddfe19bf62e43bdab5496bdc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 28 Sep 2026 08:12:43 -0400 Subject: [PATCH] fix: the single-color filter no longer takes line art for a blank image (#4483) Five of Todding's Discord doodles (pencil lines on white, 3000px) were skipped on import as "single color", so their posts showed text and no image. The predicate sampled a 64px BILINEAR thumbnail, which blends thin strokes into the paper, and 0.95 "one color" is below how white a doodle is. - Sample 256x256 by NEAREST, so each sample is a real pixel. - Default threshold 0.995: blank means essentially blank. Migration 0116 moves a stored 0.95 (the old default) with it; the settings slider now spans 0.9-1 in 0.005 steps so the value is reachable. - The Cleanup audit shares the predicate, so it stops flagging sketches too. Co-Authored-By: Claude Opus 5.5 --- .../versions/0116_single_color_threshold.py | 34 ++++++++++++++ backend/app/models/import_settings.py | 2 +- backend/app/services/audits/single_color.py | 15 +++++-- .../cleanup/SingleColorAuditCard.vue | 2 +- .../components/settings/ImportFiltersForm.vue | 4 +- frontend/src/stores/cleanup.js | 4 +- tests/test_audits_single_color.py | 45 +++++++++++++++++++ 7 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/0116_single_color_threshold.py diff --git a/alembic/versions/0116_single_color_threshold.py b/alembic/versions/0116_single_color_threshold.py new file mode 100644 index 0000000..915761d --- /dev/null +++ b/alembic/versions/0116_single_color_threshold.py @@ -0,0 +1,34 @@ +"""The single-color filter's default becomes near-total: 0.95 → 0.995. + +#4483. At 0.95 the filter rejected line art: a pencil doodle on white is +mostly white, and five of Todding's Discord doodles were skipped on import as +"single color", leaving their posts with text and no image. The predicate now +samples real pixels instead of a blurred thumbnail, and the default only +calls an image blank when it essentially is one. + +A stored 0.95 is the old default, so it moves with it. Any other value was +chosen by hand and is left alone. + +Revision ID: 0116 +Revises: 0115 +Create Date: 2026-09-28 + +""" +from alembic import op + +revision = "0116" +down_revision = "0115" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column("import_settings", "single_color_threshold", server_default="0.995") + op.execute( + "UPDATE import_settings SET single_color_threshold = 0.995 " + "WHERE single_color_threshold = 0.95" + ) + + +def downgrade() -> None: + op.alter_column("import_settings", "single_color_threshold", server_default="0.95") diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index eb340d0..c9aac02 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -39,7 +39,7 @@ class ImportSettings(Base): transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9") skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") - single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95") + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995") single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30") # Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The diff --git a/backend/app/services/audits/single_color.py b/backend/app/services/audits/single_color.py index 6bddf07..5d955db 100644 --- a/backend/app/services/audits/single_color.py +++ b/backend/app/services/audits/single_color.py @@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since 2026-07-02 — the import-side filter (Importer._single_color_hit / SkipReason.single_color), so what the audit flags and what the import skips can never disagree. + +It is meant to catch the blank: a placeholder, an error tile, a solid fill. +Line art is the case it must not catch (#4483): a pencil doodle on white can be +well over 95% white even at full size, and a smoothing downsample blends its strokes +into the paper until it measures as blank. So the sample is taken by NEAREST +(each sampled pixel is a real pixel, strokes keep their contrast), at 256px +so thin strokes are still hit, and the default threshold is near-total +(0.995) — a few percent of ink is a drawing, not an empty image. """ from PIL import Image -_THUMB_SIZE = (64, 64) +_THUMB_SIZE = (256, 256) def evaluate( @@ -20,7 +28,8 @@ def evaluate( """True iff the fraction of pixels within `tolerance` (Euclidean RGB distance) of the dominant color exceeds `threshold`. - Downsamples to 64x64 for speed (~4ms regardless of source size). + Samples 256x256 by NEAREST (see the module docstring for why not a + smoothing resample). Alpha channels are stripped; only RGB is considered. Animated images use frame 0 (PIL's default after Image.open without seek). """ @@ -30,7 +39,7 @@ def evaluate( elif im.mode not in ("RGB", "L"): im = im.convert("RGB") if im.size != _THUMB_SIZE: - im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR) + im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST) pixels = list(im.getdata()) if not pixels: return False diff --git a/frontend/src/components/cleanup/SingleColorAuditCard.vue b/frontend/src/components/cleanup/SingleColorAuditCard.vue index 1a87772..5b16302 100644 --- a/frontend/src/components/cleanup/SingleColorAuditCard.vue +++ b/frontend/src/components/cleanup/SingleColorAuditCard.vue @@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue' import { useCleanupStore } from '../../stores/cleanup.js' const store = useCleanupStore() -const threshold = ref(0.95) +const threshold = ref(0.995) const tolerance = ref(30) const audit = ref(null) const busy = ref(false) diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue index 176def1..84a31e0 100644 --- a/frontend/src/components/settings/ImportFiltersForm.vue +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -77,7 +77,7 @@ @@ -142,7 +142,7 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' } const local = reactive({ min_width: 0, min_height: 0, skip_transparent: false, transparency_threshold: 0.9, - skip_single_color: false, single_color_threshold: 0.95, + skip_single_color: false, single_color_threshold: 0.995, phash_threshold: 24, wip_title_tagging_enabled: true, }) diff --git a/frontend/src/stores/cleanup.js b/frontend/src/stores/cleanup.js index 82813a2..2edf8c0 100644 --- a/frontend/src/stores/cleanup.js +++ b/frontend/src/stores/cleanup.js @@ -12,7 +12,7 @@ export const useCleanupStore = defineStore('cleanup', () => { min_width: 0, min_height: 0, transparency_threshold: 0.9, - single_color_threshold: 0.95, + single_color_threshold: 0.995, single_color_tolerance: 30, }) @@ -24,7 +24,7 @@ export const useCleanupStore = defineStore('cleanup', () => { min_width: s.min_width ?? 0, min_height: s.min_height ?? 0, transparency_threshold: s.transparency_threshold ?? 0.9, - single_color_threshold: s.single_color_threshold ?? 0.95, + single_color_threshold: s.single_color_threshold ?? 0.995, single_color_tolerance: s.single_color_tolerance ?? 30, } } diff --git a/tests/test_audits_single_color.py b/tests/test_audits_single_color.py index e133bab..bb346f1 100644 --- a/tests/test_audits_single_color.py +++ b/tests/test_audits_single_color.py @@ -40,3 +40,48 @@ def test_single_color_evaluate_handles_rgba_input(): # Alpha channel should be ignored — only RGB matters for the rule. im = Image.new("RGBA", (50, 50), (100, 100, 100, 128)) assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True + + +# -- line art is a drawing, not a blank (#4483) --------------------------------- + +_T = 0.995 # the default since alembic 0116 +_TOL = 30 + + +def _doodle(size=(1500, 2000), lines=12): + """Pencil line art on white: a few thin dark strokes plus faint + construction lines — a couple of percent ink, the rest paper.""" + from PIL import ImageDraw + + im = Image.new("RGB", size, (255, 255, 255)) + draw = ImageDraw.Draw(im) + w, h = size + for i in range(lines): + x = (i + 1) * w // (lines + 1) + draw.line([(x, 0), (w - x, h)], fill=(70, 60, 60), width=3) + draw.line([(0, (i + 1) * h // (lines + 1)), (w, h - x)], fill=(235, 235, 235), width=2) + return im + + +def test_line_art_on_white_is_not_single_color(): + """Five of Todding's Discord doodles were skipped as blank: a smoothing + 64px thumbnail blended the strokes into the paper.""" + assert single_color.evaluate(_doodle(), threshold=_T, tolerance=_TOL) is False + + +def test_a_blank_page_with_a_small_mark_is_still_single_color(): + from PIL import ImageDraw + + im = Image.new("RGB", (1000, 1000), (250, 250, 250)) + ImageDraw.Draw(im).rectangle([480, 480, 520, 520], fill=(40, 40, 40)) + assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True + + +def test_a_noisy_solid_fill_is_still_single_color(): + """JPEG noise on a placeholder stays inside the tolerance.""" + im = Image.new("RGB", (600, 600), (30, 90, 160)) + for i in range(0, 600 * 600, 7): + x, y = i % 600, i // 600 + d = (i % 21) - 10 + im.putpixel((x, y), (30 + d, 90 + d, 160 + d)) + assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True