diff --git a/backend/app/services/artist_directory_service.py b/backend/app/services/artist_directory_service.py index a16c1aa..eda9afc 100644 --- a/backend/app/services/artist_directory_service.py +++ b/backend/app/services/artist_directory_service.py @@ -127,17 +127,20 @@ class ArtistDirectoryService: ImageRecord.artist_id.label("artist_id"), ImageRecord.sha256.label("sha256"), ImageRecord.mime.label("mime"), + ImageRecord.thumbnail_path.label("thumbnail_path"), rn, ) .where(ImageRecord.artist_id.in_(artist_ids)) .subquery() ) stmt = ( - select(sub.c.artist_id, sub.c.sha256, sub.c.mime) + select( + sub.c.artist_id, sub.c.sha256, sub.c.mime, sub.c.thumbnail_path, + ) .where(sub.c.rn <= _PREVIEW_COUNT) .order_by(sub.c.artist_id, sub.c.rn) ) out: dict[int, list[str]] = {} - for aid, sha, mime in (await self.session.execute(stmt)).all(): - out.setdefault(aid, []).append(thumbnail_url(sha, mime)) + for aid, sha, mime, tp in (await self.session.execute(stmt)).all(): + out.setdefault(aid, []).append(thumbnail_url(tp, sha, mime)) return out diff --git a/backend/app/services/artist_service.py b/backend/app/services/artist_service.py index c6aff2b..56f19ff 100644 --- a/backend/app/services/artist_service.py +++ b/backend/app/services/artist_service.py @@ -198,7 +198,7 @@ class ArtistService: "mime": r.mime, "width": r.width, "height": r.height, - "thumbnail_url": thumbnail_url(r.sha256, r.mime), + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), } for r in rows ], diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 0a46fec..095890d 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -90,9 +90,27 @@ class TimelineBucket: count: int -def thumbnail_url(sha256_hex: str, mime: str) -> str: - # Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go - # under /images/thumbs/. The MIME determines the extension. +def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str: + """Return the URL to fetch a thumbnail. + + Prefers the stored thumbnail_path verbatim — Quart serves /images/* + 1:1 from the volume (frontend.py:20-36), so the URL IS the disk + path. Falls back to deriving from (sha256, mime) only when the + record's thumbnail_path is NULL (thumbnailer hasn't run yet); that + URL will 404 until backfill catches it, same as before the path + was tracked. + + Pre-2026-05-30 this was derived only from (sha256, mime), which + disagreed with the actual on-disk extension when the thumbnailer + chose its format from transparency rather than MIME — every PNG + source without alpha (extension was .jpg on disk) and every WebP + source with alpha (extension was .png on disk) silently 404'd + despite the thumbnail file existing. + """ + if thumbnail_path: + return thumbnail_path + # Fallback for records with no thumbnail recorded yet — preserves + # prior behavior (URL exists but 404s until backfill regenerates). ext = ".png" if mime in ("image/png", "image/gif") else ".jpg" bucket = sha256_hex[:3] return f"/images/thumbs/{bucket}/{sha256_hex}{ext}" @@ -198,7 +216,7 @@ class GalleryService: created_at=record.created_at, effective_date=eff_date, posted_at=posted_at, - thumbnail_url=thumbnail_url(record.sha256, record.mime), + thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime), artist=artists.get(record.id), ) for record, posted_at, eff_date in rows @@ -306,7 +324,7 @@ class GalleryService: "integrity_status": record.integrity_status, "created_at": record.created_at.isoformat(), "posted_at": posted_at.isoformat() if posted_at else None, - "thumbnail_url": thumbnail_url(record.sha256, record.mime), + "thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime), "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", "artist": ( {"id": artist.id, "name": artist.name, "slug": artist.slug} diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index 319d645..634cfd3 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -207,6 +207,7 @@ class PostFeedService: ImageRecord.primary_post_id, ImageRecord.sha256, ImageRecord.mime, + ImageRecord.thumbnail_path, func.row_number().over( partition_by=ImageRecord.primary_post_id, order_by=ImageRecord.id.asc(), @@ -220,18 +221,18 @@ class PostFeedService: ) stmt = select( ranked.c.id, ranked.c.primary_post_id, - ranked.c.sha256, ranked.c.mime, ranked.c.total, + ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total, ) if limit is not None: stmt = stmt.where(ranked.c.rn <= limit) rows = (await self.session.execute(stmt)).all() out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids} - for img_id, pid, sha, mime, total in rows: + for img_id, pid, sha, mime, tp, total in rows: entry = out.setdefault(pid, {"thumbs": [], "more": 0}) entry["thumbs"].append({ "image_id": img_id, - "thumbnail_url": thumbnail_url(sha, mime), + "thumbnail_url": thumbnail_url(tp, sha, mime), "mime": mime, }) # `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT. diff --git a/backend/app/services/series_service.py b/backend/app/services/series_service.py index 6ed3df8..361caa5 100644 --- a/backend/app/services/series_service.py +++ b/backend/app/services/series_service.py @@ -63,6 +63,7 @@ class SeriesService: ImageRecord.sha256, ImageRecord.mime, ImageRecord.path, + ImageRecord.thumbnail_path, ) .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) .where(SeriesPage.series_tag_id == series_tag_id) @@ -75,7 +76,7 @@ class SeriesService: { "image_id": r.image_id, "page_number": r.page_number, - "thumbnail_url": thumbnail_url(r.sha256, r.mime), + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", } for r in rows diff --git a/backend/app/services/showcase_service.py b/backend/app/services/showcase_service.py index b45e0b5..f34051b 100644 --- a/backend/app/services/showcase_service.py +++ b/backend/app/services/showcase_service.py @@ -34,7 +34,7 @@ class ShowcaseService: "mime": r.mime, "width": r.width, "height": r.height, - "thumbnail_url": thumbnail_url(r.sha256, r.mime), + "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime), } for r in rows ] diff --git a/backend/app/services/tag_directory_service.py b/backend/app/services/tag_directory_service.py index a0ffaaa..c88b04c 100644 --- a/backend/app/services/tag_directory_service.py +++ b/backend/app/services/tag_directory_service.py @@ -115,12 +115,17 @@ class TagDirectoryService: .subquery() ) stmt = ( - select(sub.c.tag_id, ImageRecord.sha256, ImageRecord.mime) + select( + sub.c.tag_id, + ImageRecord.sha256, + ImageRecord.mime, + ImageRecord.thumbnail_path, + ) .join(ImageRecord, ImageRecord.id == sub.c.image_record_id) .where(sub.c.rn <= 3) .order_by(sub.c.tag_id, sub.c.rn) ) out: dict[int, list[str]] = {} - for tag_id, sha, mime in (await self.session.execute(stmt)).all(): - out.setdefault(tag_id, []).append(thumbnail_url(sha, mime)) + for tag_id, sha, mime, tp in (await self.session.execute(stmt)).all(): + out.setdefault(tag_id, []).append(thumbnail_url(tp, sha, mime)) return out diff --git a/backend/app/utils/_archive_probe_runner.py b/backend/app/utils/_archive_probe_runner.py new file mode 100644 index 0000000..f4b0c88 --- /dev/null +++ b/backend/app/utils/_archive_probe_runner.py @@ -0,0 +1,33 @@ +"""Subprocess entrypoint for safe_probe.probe_archive — see safe_probe.py. + +probe_archive spawns this via subprocess (not multiprocessing.Process) +because Celery's prefork worker pool runs tasks in DAEMON processes and +Python's multiprocessing forbids daemon processes from spawning children +("AssertionError: daemonic processes are not allowed to have children", +operator-flagged 2026-05-30 — every archive import failed at task +startup). subprocess has no such restriction; we still get crash- +isolation because a probe segfault/OOM exits non-zero rather than +killing the worker. + +Prints a single JSON line on stdout: {"status": "ok"|"error", +"detail": "..."?}. Exit code 0 for clean outcomes; non-zero exit +(signal / OOM-kill / unhandled exception) is the poison-pill signature +the parent maps to ProbeResult(crashed=True). +""" +import json +import sys + +from .safe_probe import _run_probe + + +def main() -> int: + if len(sys.argv) != 2: + print(json.dumps({"status": "error", "detail": "usage: "})) + return 2 + status, detail = _run_probe(sys.argv[1]) + print(json.dumps({"status": status, "detail": detail})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/app/utils/safe_probe.py b/backend/app/utils/safe_probe.py index 02c387c..d8e5360 100644 --- a/backend/app/utils/safe_probe.py +++ b/backend/app/utils/safe_probe.py @@ -28,8 +28,8 @@ Operator-requested 2026-05-28 (Layer 3). """ import json -import multiprocessing as mp import subprocess +import sys from dataclasses import dataclass from pathlib import Path @@ -40,6 +40,13 @@ ARCHIVE_PROBE_TIMEOUT_SECONDS = 120 # art-pack archives while stopping a few-KB zip that expands to TB). MAX_ARCHIVE_UNCOMPRESSED_BYTES = 4 * 1024 * 1024 * 1024 +# Repo root for the subprocess cwd so `python -m backend.app.utils.*` +# resolves regardless of where Celery / pytest started. backend/app/utils +# = parents[0]; backend/app = parents[1]; backend = parents[2]; repo root +# = parents[3]. +_REPO_ROOT = Path(__file__).resolve().parents[3] +_PROBE_RUNNER_MODULE = "backend.app.utils._archive_probe_runner" + @dataclass(frozen=True) class ProbeResult: @@ -91,54 +98,68 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> def probe_archive(path: Path, *, timeout: float = ARCHIVE_PROBE_TIMEOUT_SECONDS) -> ProbeResult: - """Bomb-size guard + isolated integrity test for an archive.""" - ctx = mp.get_context("spawn") - q = ctx.Queue() - proc = ctx.Process(target=_archive_probe_target, args=(str(path), q)) - proc.start() - proc.join(timeout) - if proc.is_alive(): - proc.terminate() - proc.join(5) + """Bomb-size guard + isolated integrity test for an archive. + + Runs via subprocess (not multiprocessing.Process) because Celery's + prefork worker pool is daemon-mode and Python's multiprocessing + forbids daemon processes from spawning children ("AssertionError: + daemonic processes are not allowed to have children"). subprocess + has no such restriction and still gives the crash isolation: a probe + segfault/OOM exits non-zero rather than killing the worker. + """ + try: + result = subprocess.run( + [sys.executable, "-m", _PROBE_RUNNER_MODULE, str(path)], + capture_output=True, text=True, timeout=timeout, + cwd=str(_REPO_ROOT), + ) + except subprocess.TimeoutExpired: return ProbeResult(ok=False, crashed=True, reason="archive probe timed out") - if proc.exitcode != 0: - # Negative exitcode = killed by signal (segfault); positive = - # the child os._exit'd or was OOM-killed. Either way the file - # hard-crashed the probe — the poison-pill signature. + if result.returncode != 0: + # Negative = killed by signal (segfault); positive = unhandled + # exception or OOM-kill. Either way: poison-pill signature. return ProbeResult( ok=False, crashed=True, - reason=f"archive probe crashed (exit {proc.exitcode})", + reason=f"archive probe crashed (exit {result.returncode})", ) + last_line = result.stdout.strip().splitlines()[-1:] or [""] try: - outcome = q.get(timeout=5) - except Exception: # noqa: BLE001 — empty queue / broken pipe - return ProbeResult(ok=False, crashed=True, reason="archive probe produced no result") - status, detail = outcome - if status == "ok": + outcome = json.loads(last_line[0]) + except json.JSONDecodeError as exc: + return ProbeResult( + ok=False, crashed=True, + reason=f"archive probe produced no parseable result: {exc}", + ) + if outcome.get("status") == "ok": return ProbeResult(ok=True) - return ProbeResult(ok=False, crashed=False, reason=detail) + return ProbeResult( + ok=False, crashed=False, + reason=outcome.get("detail") or "archive probe rejected", + ) -def _archive_probe_target(path_str: str, q) -> None: - """Runs in the spawned child. Reads member sizes (bomb guard) then - runs the format's integrity test. Puts ('ok', None) or - ('error', reason). A crash/OOM here never reaches the queue — the - parent reads the non-zero exit code instead.""" +def _run_probe(path_str: str) -> tuple[str, str | None]: + """Pure-Python body of the archive probe — bomb-guard + integrity test. + + Returns ('ok', None) or ('error', reason). Caught exceptions become + clean 'error' rejections; uncaught crashes in the subprocess become + non-zero exit codes (poison-pill signature) handled by probe_archive. + + Exposed at the module level so the subprocess runner and tests both + call the same code path. + """ path = Path(path_str) ext = path.suffix.lower() try: total, test_bad = _inspect_archive(path, ext) except Exception as exc: # noqa: BLE001 — clean rejection - q.put(("error", f"{type(exc).__name__}: {exc}")) - return + return ("error", f"{type(exc).__name__}: {exc}") if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES: gib = total / (1024 ** 3) - q.put(("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap")) - return + return ("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap") if test_bad is not None: - q.put(("error", f"integrity test failed at member {test_bad!r}")) - return - q.put(("ok", None)) + return ("error", f"integrity test failed at member {test_bad!r}") + return ("ok", None) def _inspect_archive(path: Path, ext: str): diff --git a/frontend/src/stores/gallery.js b/frontend/src/stores/gallery.js index 0c4ca77..1a8245a 100644 --- a/frontend/src/stores/gallery.js +++ b/frontend/src/stores/gallery.js @@ -2,6 +2,13 @@ import { defineStore } from 'pinia' import { ref, computed } from 'vue' import { useApi } from '../composables/useApi.js' +// Operator-confirmed 2026-05-30: fetch PAGE-sized chunks instead of one +// 50-item request so items render as each batch lands. Total initial +// count is unchanged (PAGE * INITIAL_BATCHES = 50). Infinite-scroll also +// pulls PAGE per trigger to keep appends progressive. +const PAGE = 5 +const INITIAL_BATCHES = 10 + export const useGalleryStore = defineStore('gallery', () => { const api = useApi() @@ -21,7 +28,13 @@ export const useGalleryStore = defineStore('gallery', () => { images.value = [] dateGroups.value = [] nextCursor.value = null - await loadMore() + // Sequentially fetch INITIAL_BATCHES chunks so items render as each + // batch lands rather than blocking on one big response. Stop early + // when the backend reports no more pages. + for (let i = 0; i < INITIAL_BATCHES; i++) { + if (i > 0 && nextCursor.value === null) break + await loadMore() + } } async function loadMore() { @@ -30,7 +43,7 @@ export const useGalleryStore = defineStore('gallery', () => { error.value = null const myId = ++inflightId try { - const params = { limit: 50, ...activeFilterParam() } + const params = { limit: PAGE, ...activeFilterParam() } if (nextCursor.value) params.cursor = nextCursor.value const body = await api.get('/api/gallery/scroll', { params }) if (myId !== inflightId) return // stale response diff --git a/frontend/src/stores/showcase.js b/frontend/src/stores/showcase.js index 2b93d3d..5e7686e 100644 --- a/frontend/src/stores/showcase.js +++ b/frontend/src/stores/showcase.js @@ -3,7 +3,13 @@ import { ref, computed } from 'vue' import { useApi } from '../composables/useApi.js' import { useAsyncAction } from '../composables/useAsyncAction.js' -const PAGE = 60 +// Operator-confirmed 2026-05-30: instead of one 60-item request, fetch +// PAGE-sized chunks sequentially so items render as each batch lands +// rather than blocking on the full 60-item response. Total initial count +// is unchanged (PAGE * INITIAL_BATCHES = 60). Infinite-scroll also pulls +// PAGE items per trigger so subsequent appends stay progressive too. +const PAGE = 5 +const INITIAL_BATCHES = 12 export const useShowcaseStore = defineStore('showcase', () => { const api = useApi() @@ -23,11 +29,21 @@ export const useShowcaseStore = defineStore('showcase', () => { }) } - async function shuffle() { + // Reset state and fetch INITIAL_BATCHES chunks in sequence. Used by + // mount, the Shuffle button, and the R-key handler — all want the + // same progressive-cascade behavior. + async function loadInitial() { images.value = [] seen.clear() exhausted.value = false - await fetchPage() + for (let i = 0; i < INITIAL_BATCHES; i++) { + if (exhausted.value) break + await fetchPage() + } + } + + async function shuffle() { + await loadInitial() } const hasMore = computed(() => !exhausted.value) @@ -35,5 +51,5 @@ export const useShowcaseStore = defineStore('showcase', () => { () => !loading.value && images.value.length === 0 && error.value === null ) - return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle } + return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle, loadInitial } }) diff --git a/frontend/src/views/ShowcaseView.vue b/frontend/src/views/ShowcaseView.vue index a091d84..63de41f 100644 --- a/frontend/src/views/ShowcaseView.vue +++ b/frontend/src/views/ShowcaseView.vue @@ -70,7 +70,7 @@ function onKeydown(e) { } onMounted(() => { - if (store.images.length === 0) store.fetchPage() + if (store.images.length === 0) store.loadInitial() window.addEventListener('keydown', onKeydown) }) onUnmounted(() => window.removeEventListener('keydown', onKeydown)) diff --git a/tests/test_safe_probe.py b/tests/test_safe_probe.py index 6dd7ea2..98a67b3 100644 --- a/tests/test_safe_probe.py +++ b/tests/test_safe_probe.py @@ -1,12 +1,11 @@ """Layer-3 subprocess-isolated probe tests. -The bomb-guard cap is exercised against `_archive_probe_target` directly -(in-process, where a monkeypatch on the module constant takes effect) — -spawn re-imports the module in the child, so a parent-process -monkeypatch wouldn't reach the spawned worker. +The bomb-guard cap is exercised against `_run_probe` directly (in-process, +where a monkeypatch on the module constant takes effect) — the real probe +runs in a subprocess that re-imports the module, so a parent-process +monkeypatch wouldn't reach the spawned interpreter. """ -import multiprocessing as mp import zipfile from pathlib import Path @@ -46,16 +45,14 @@ def test_inspect_archive_reports_size_and_clean_integrity(tmp_path): assert bad is None -def test_archive_probe_target_bomb_guard(tmp_path, monkeypatch): - """In-process call to the child target so the monkeypatched cap - takes effect. A normal zip whose uncompressed size exceeds the - (lowered) cap is rejected with the bomb-guard reason.""" +def test_run_probe_bomb_guard(tmp_path, monkeypatch): + """In-process call to _run_probe so the monkeypatched cap takes effect. + The public probe_archive runs in a subprocess which re-imports the + module and wouldn't see the patched constant.""" monkeypatch.setattr(safe_probe, "MAX_ARCHIVE_UNCOMPRESSED_BYTES", 10) z = tmp_path / "bomb.zip" _zip(z, {"big.txt": b"x" * 5000}) # 5000 uncompressed > 10-byte cap - q = mp.get_context("spawn").Queue() - safe_probe._archive_probe_target(str(z), q) - status, detail = q.get(timeout=5) + status, detail = safe_probe._run_probe(str(z)) assert status == "error" assert "bomb-guard cap" in detail