From 7aa7f5a3d69b2bed2bda9f2011d1c7503a09524d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:35:06 -0400 Subject: [PATCH 01/20] =?UTF-8?q?feat(thumb-backfill):=20=5Fthumb=5Fis=5Fv?= =?UTF-8?q?alid=20helper=20=E2=80=94=20JPEG/PNG=20magic-byte=20check=20on?= =?UTF-8?q?=20the=20on-disk=20thumbnail=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/tasks/thumbnail.py | 24 +++++++++++++++++++++ tests/test_backfill_thumbnails.py | 35 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/test_backfill_thumbnails.py diff --git a/backend/app/tasks/thumbnail.py b/backend/app/tasks/thumbnail.py index dec644f..ab465ce 100644 --- a/backend/app/tasks/thumbnail.py +++ b/backend/app/tasks/thumbnail.py @@ -17,6 +17,30 @@ from ._sync_engine import sync_session_factory as _sync_session_factory IMAGES_ROOT = Path("/images") +THUMB_MAGIC_JPEG = b"\xff\xd8\xff" +THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n" + + +def _thumb_is_valid(path: Path) -> bool: + """Return True iff `path` exists and starts with a JPEG or PNG magic header. + + The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for + opaque sources, PNG for alpha sources. Anything else (missing file, OSError, + truncated, wrong magic) is invalid. + """ + try: + with path.open("rb") as f: + head = f.read(12) + except OSError: + return False + if len(head) < 8: + return False + if head[:3] == THUMB_MAGIC_JPEG: + return True + if head[:8] == THUMB_MAGIC_PNG: + return True + return False + @celery.task( name="backend.app.tasks.thumbnail.generate_thumbnail", diff --git a/tests/test_backfill_thumbnails.py b/tests/test_backfill_thumbnails.py new file mode 100644 index 0000000..5537577 --- /dev/null +++ b/tests/test_backfill_thumbnails.py @@ -0,0 +1,35 @@ +"""Thumbnail backfill: _thumb_is_valid helper + backfill_thumbnails planner.""" + +import pytest + +from backend.app.tasks.thumbnail import _thumb_is_valid + +pytestmark = pytest.mark.integration + + +def test_thumb_is_valid_jpeg(tmp_path): + p = tmp_path / "good.jpg" + p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100) + assert _thumb_is_valid(p) is True + + +def test_thumb_is_valid_png(tmp_path): + p = tmp_path / "good.png" + p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100) + assert _thumb_is_valid(p) is True + + +def test_thumb_is_valid_too_short(tmp_path): + p = tmp_path / "tiny" + p.write_bytes(b"\xff\xd8") + assert _thumb_is_valid(p) is False + + +def test_thumb_is_valid_wrong_magic(tmp_path): + p = tmp_path / "garbage" + p.write_bytes(b"\x00" * 12) + assert _thumb_is_valid(p) is False + + +def test_thumb_is_valid_missing_file(tmp_path): + assert _thumb_is_valid(tmp_path / "nope") is False From a41eddae3f042bfea1ffcfe8c36183a8ca03403f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:36:21 -0400 Subject: [PATCH 02/20] =?UTF-8?q?feat(thumb-backfill):=20backfill=5Fthumbn?= =?UTF-8?q?ails=20planner=20task=20=E2=80=94=20keyset-paginates=20ImageRec?= =?UTF-8?q?ord,=20NULLs=20bad=20thumb=20paths,=20enqueues=20generate=5Fthu?= =?UTF-8?q?mbnail=20for=20NULL/missing/corrupt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/tasks/thumbnail.py | 55 ++++++++ tests/test_backfill_thumbnails.py | 207 ++++++++++++++++++++++++++++++ tests/test_tasks_register.py | 4 + 3 files changed, 266 insertions(+) diff --git a/backend/app/tasks/thumbnail.py b/backend/app/tasks/thumbnail.py index ab465ce..0061bb1 100644 --- a/backend/app/tasks/thumbnail.py +++ b/backend/app/tasks/thumbnail.py @@ -74,3 +74,58 @@ def generate_thumbnail(self, image_id: int) -> dict: session.add(record) session.commit() return {"status": "ok", "image_id": image_id, "path": str(result.path)} + + +@celery.task( + name="backend.app.tasks.thumbnail.backfill_thumbnails", + bind=True, +) +def backfill_thumbnails(self) -> dict: + """Scan ImageRecord and enqueue generate_thumbnail for rows whose + thumbnail is missing, gone from disk, or has wrong magic bytes. + + Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for + rows that point at a missing or corrupt file before enqueueing — keeps + the DB self-consistent on partial runs and makes re-runs safe. + + Returns {"enqueued": N, "ok": M, "regenerated": K} where: + - enqueued = total generate_thumbnail.delay() calls + - ok = rows whose existing thumbnail file is valid (skipped) + - regenerated = subset of enqueued that had a non-NULL thumbnail_path + cleared (i.e. missing + corrupt) + """ + from sqlalchemy import select, update + + SessionLocal = _sync_session_factory() + enqueued = 0 + ok = 0 + regenerated = 0 + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord.id, ImageRecord.thumbnail_path) + .where(ImageRecord.id > last_id) + .order_by(ImageRecord.id.asc()) + .limit(500) + ).all() + if not rows: + break + for image_id, thumb_path in rows: + if thumb_path is None: + generate_thumbnail.delay(image_id) + enqueued += 1 + elif _thumb_is_valid(Path(thumb_path)): + ok += 1 + else: + session.execute( + update(ImageRecord) + .where(ImageRecord.id == image_id) + .values(thumbnail_path=None) + ) + generate_thumbnail.delay(image_id) + enqueued += 1 + regenerated += 1 + session.commit() + last_id = rows[-1][0] + return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated} diff --git a/tests/test_backfill_thumbnails.py b/tests/test_backfill_thumbnails.py index 5537577..faf2a77 100644 --- a/tests/test_backfill_thumbnails.py +++ b/tests/test_backfill_thumbnails.py @@ -1,7 +1,10 @@ """Thumbnail backfill: _thumb_is_valid helper + backfill_thumbnails planner.""" +from pathlib import Path + import pytest +from backend.app.models import ImageRecord from backend.app.tasks.thumbnail import _thumb_is_valid pytestmark = pytest.mark.integration @@ -33,3 +36,207 @@ def test_thumb_is_valid_wrong_magic(tmp_path): def test_thumb_is_valid_missing_file(tmp_path): assert _thumb_is_valid(tmp_path / "nope") is False + + +# --- backfill_thumbnails planner tests ------------------------------------ + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + """sessionmaker-like returning the test's bound session, matching the + pattern in tests/test_backfill_phash.py.""" + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _sha(prefix: str) -> str: + return f"{prefix}".ljust(64, "0")[:64] + + +def _rec(db_sync, path, *, sha, thumb_path=None, mime="image/jpeg"): + rec = ImageRecord( + path=str(path), sha256=sha, size_bytes=1, mime=mime, + width=64, height=64, origin="imported_filesystem", + integrity_status="unknown", + thumbnail_path=str(thumb_path) if thumb_path is not None else None, + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def _write_jpeg(p: Path) -> Path: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100) + return p + + +def _write_png(p: Path) -> Path: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100) + return p + + +def _write_garbage(p: Path) -> Path: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"\x00" * 12) + return p + + +def test_backfill_null_path_enqueued(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src = tmp_path / "a.bin" + src.write_bytes(b"x") + rec = _rec(db_sync, src, sha=_sha("a"), thumb_path=None) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + assert result == {"enqueued": 1, "ok": 0, "regenerated": 0} + assert delayed == [rec.id] + + +def test_backfill_missing_file_clears_and_enqueues(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src = tmp_path / "b.bin" + src.write_bytes(b"x") + rec = _rec( + db_sync, src, sha=_sha("b"), + thumb_path=tmp_path / "thumbs" / "missing.jpg", + ) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + db_sync.expire_all() + assert result == {"enqueued": 1, "ok": 0, "regenerated": 1} + assert delayed == [rec.id] + assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None + + +def test_backfill_valid_jpeg_skipped(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src = tmp_path / "c.bin" + src.write_bytes(b"x") + thumb = _write_jpeg(tmp_path / "thumbs" / "c.jpg") + rec = _rec(db_sync, src, sha=_sha("c"), thumb_path=thumb) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + db_sync.expire_all() + assert result == {"enqueued": 0, "ok": 1, "regenerated": 0} + assert delayed == [] + assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb) + + +def test_backfill_valid_png_skipped(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src = tmp_path / "d.bin" + src.write_bytes(b"x") + thumb = _write_png(tmp_path / "thumbs" / "d.png") + _rec(db_sync, src, sha=_sha("d"), thumb_path=thumb) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + assert result == {"enqueued": 0, "ok": 1, "regenerated": 0} + assert delayed == [] + + +def test_backfill_corrupt_magic_clears_and_enqueues(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src = tmp_path / "e.bin" + src.write_bytes(b"x") + thumb = _write_garbage(tmp_path / "thumbs" / "e.jpg") + rec = _rec(db_sync, src, sha=_sha("e"), thumb_path=thumb) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + db_sync.expire_all() + assert result == {"enqueued": 1, "ok": 0, "regenerated": 1} + assert delayed == [rec.id] + assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None + + +def test_backfill_mixed_aggregate(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import thumbnail as m + + src_null = tmp_path / "src_null.bin" + src_null.write_bytes(b"x") + src_jpeg = tmp_path / "src_jpeg.bin" + src_jpeg.write_bytes(b"x") + src_png = tmp_path / "src_png.bin" + src_png.write_bytes(b"x") + src_missing = tmp_path / "src_missing.bin" + src_missing.write_bytes(b"x") + src_bad = tmp_path / "src_bad.bin" + src_bad.write_bytes(b"x") + + jpeg = _write_jpeg(tmp_path / "thumbs" / "ok.jpg") + png = _write_png(tmp_path / "thumbs" / "ok.png") + bad = _write_garbage(tmp_path / "thumbs" / "bad.jpg") + + r_null = _rec(db_sync, src_null, sha=_sha("aa"), thumb_path=None) + _rec(db_sync, src_jpeg, sha=_sha("bb"), thumb_path=jpeg) + _rec(db_sync, src_png, sha=_sha("cc"), thumb_path=png) + r_missing = _rec( + db_sync, src_missing, sha=_sha("dd"), + thumb_path=tmp_path / "thumbs" / "missing.jpg", + ) + r_bad = _rec(db_sync, src_bad, sha=_sha("ee"), thumb_path=bad) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + delayed: list[int] = [] + monkeypatch.setattr( + m.generate_thumbnail, "delay", lambda image_id: delayed.append(image_id) + ) + + result = m.backfill_thumbnails() + assert result == {"enqueued": 3, "ok": 2, "regenerated": 2} + assert sorted(delayed) == sorted([r_null.id, r_missing.id, r_bad.id]) diff --git a/tests/test_tasks_register.py b/tests/test_tasks_register.py index a8d4f72..11f457c 100644 --- a/tests/test_tasks_register.py +++ b/tests/test_tasks_register.py @@ -23,3 +23,7 @@ def test_import_media_file_registered(): def test_generate_thumbnail_registered(): assert "backend.app.tasks.thumbnail.generate_thumbnail" in celery.tasks + + +def test_backfill_thumbnails_registered(): + assert "backend.app.tasks.thumbnail.backfill_thumbnails" in celery.tasks From 983da9e5b144634874ddc252b479f6293b6b4333 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:36:38 -0400 Subject: [PATCH 03/20] =?UTF-8?q?feat(thumb-backfill):=20/api/thumbnails/b?= =?UTF-8?q?ackfill=20endpoint=20=E2=80=94=20POST=20=E2=86=92=20202=20+=20c?= =?UTF-8?q?elery=5Ftask=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/__init__.py | 2 ++ backend/app/api/thumbnails.py | 13 +++++++++++++ tests/test_api_thumbnails.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 backend/app/api/thumbnails.py create mode 100644 tests/test_api_thumbnails.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 4a88c1e..b8dc7c4 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -38,6 +38,7 @@ def all_blueprints() -> list[Blueprint]: from .system_activity import system_activity_bp from .system_backup import system_backup_bp from .tags import tags_bp + from .thumbnails import thumbnails_bp return [ api_bp, attachments_bp, @@ -58,6 +59,7 @@ def all_blueprints() -> list[Blueprint]: allowlist_bp, aliases_bp, ml_admin_bp, + thumbnails_bp, sources_bp, platforms_bp, posts_bp, diff --git a/backend/app/api/thumbnails.py b/backend/app/api/thumbnails.py new file mode 100644 index 0000000..eff1491 --- /dev/null +++ b/backend/app/api/thumbnails.py @@ -0,0 +1,13 @@ +"""Thumbnail admin API: backfill trigger.""" + +from quart import Blueprint, jsonify + +thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails") + + +@thumbnails_bp.route("/backfill", methods=["POST"]) +async def trigger_backfill(): + from ..tasks.thumbnail import backfill_thumbnails + + r = backfill_thumbnails.delay() + return jsonify({"celery_task_id": r.id}), 202 diff --git a/tests/test_api_thumbnails.py b/tests/test_api_thumbnails.py new file mode 100644 index 0000000..b9d07a4 --- /dev/null +++ b/tests/test_api_thumbnails.py @@ -0,0 +1,32 @@ +import pytest + +from backend.app import create_app +from backend.app.celery_app import celery + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_trigger_thumbnail_backfill(client): + r = await client.post("/api/thumbnails/backfill") + assert r.status_code == 202 + body = await r.get_json() + assert "celery_task_id" in body From ecd0199799df45d955ce87e9bb475b21fdd12d24 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:36:45 -0400 Subject: [PATCH 04/20] =?UTF-8?q?feat(thumb-backfill):=20Pinia=20store=20?= =?UTF-8?q?=E2=80=94=20triggerBackfill()=20POSTs=20/api/thumbnails/backfil?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/stores/thumbnails.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 frontend/src/stores/thumbnails.js diff --git a/frontend/src/stores/thumbnails.js b/frontend/src/stores/thumbnails.js new file mode 100644 index 0000000..5f86f2b --- /dev/null +++ b/frontend/src/stores/thumbnails.js @@ -0,0 +1,12 @@ +import { defineStore } from 'pinia' +import { useApi } from '../composables/useApi.js' + +export const useThumbnailsStore = defineStore('thumbnails', () => { + const api = useApi() + + async function triggerBackfill () { + await api.post('/api/thumbnails/backfill') + } + + return { triggerBackfill } +}) From b7f693b15ebf27f11dd38809554059052f788eec Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:36:55 -0400 Subject: [PATCH 05/20] =?UTF-8?q?feat(thumb-backfill):=20ThumbnailBackfill?= =?UTF-8?q?Card=20=E2=80=94=20'Run=20backfill=20now'=20button,=20mirrors?= =?UTF-8?q?=20MLBackfillCard=20pattern?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../settings/ThumbnailBackfillCard.vue | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 frontend/src/components/settings/ThumbnailBackfillCard.vue diff --git a/frontend/src/components/settings/ThumbnailBackfillCard.vue b/frontend/src/components/settings/ThumbnailBackfillCard.vue new file mode 100644 index 0000000..489c45f --- /dev/null +++ b/frontend/src/components/settings/ThumbnailBackfillCard.vue @@ -0,0 +1,30 @@ + + + From 06913eba8e5a28b614d69ec63a5aa5a89c95309b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 10:37:09 -0400 Subject: [PATCH 06/20] =?UTF-8?q?feat(thumb-backfill):=20MaintenancePanel?= =?UTF-8?q?=20=E2=80=94=20wire=20ThumbnailBackfillCard=20into=20grid,=20br?= =?UTF-8?q?oaden=20intro=20to=20cover=20non-ML=20backfills?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/settings/MaintenancePanel.vue | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 90c410b..42c775d 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -1,13 +1,15 @@