bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Merge pull request 'Thumbnail URL fix + archive daemon fix + batched initial loads' (#37) from dev into main

Browse Source
This commit was merged in pull request #37.
This commit is contained in:
bvandeusen
2026-05-30 22:01:43 -04:00
parent ed358757dc 810baf63ac
commit 237575447d
13 changed files with 177 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/artist_directory_service.py
+6 -3
View File
@@ -127,17 +127,20 @@ class ArtistDirectoryService:
ImageRecord.artist_id.label("artist_id"),
ImageRecord.sha256.label("sha256"),
ImageRecord.mime.label("mime"),
ImageRecord.thumbnail_path.label("thumbnail_path"),
rn,
)
.where(ImageRecord.artist_id.in_(artist_ids))
.subquery()
)
stmt = (
select(sub.c.artist_id, sub.c.sha256, sub.c.mime)
select(
sub.c.artist_id, sub.c.sha256, sub.c.mime, sub.c.thumbnail_path,
)
.where(sub.c.rn <= _PREVIEW_COUNT)
.order_by(sub.c.artist_id, sub.c.rn)
)
out: dict[int, list[str]] = {}
for aid, sha, mime in (await self.session.execute(stmt)).all():
out.setdefault(aid, []).append(thumbnail_url(sha, mime))
for aid, sha, mime, tp in (await self.session.execute(stmt)).all():
out.setdefault(aid, []).append(thumbnail_url(tp, sha, mime))
return out
backend/app/services/artist_service.py
+1 -1
View File
@@ -198,7 +198,7 @@ class ArtistService:
"mime": r.mime,
"width": r.width,
"height": r.height,
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
}
for r in rows
],
backend/app/services/gallery_service.py
+23 -5
View File
@@ -90,9 +90,27 @@ class TimelineBucket:
count: int
def thumbnail_url(sha256_hex: str, mime: str) -> str:
# Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go
# under /images/thumbs/. The MIME determines the extension.
def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str:
"""Return the URL to fetch a thumbnail.
Prefers the stored thumbnail_path verbatim — Quart serves /images/*
1:1 from the volume (frontend.py:20-36), so the URL IS the disk
path. Falls back to deriving from (sha256, mime) only when the
record's thumbnail_path is NULL (thumbnailer hasn't run yet); that
URL will 404 until backfill catches it, same as before the path
was tracked.
Pre-2026-05-30 this was derived only from (sha256, mime), which
disagreed with the actual on-disk extension when the thumbnailer
chose its format from transparency rather than MIME — every PNG
source without alpha (extension was .jpg on disk) and every WebP
source with alpha (extension was .png on disk) silently 404'd
despite the thumbnail file existing.
"""
if thumbnail_path:
return thumbnail_path
# Fallback for records with no thumbnail recorded yet — preserves
# prior behavior (URL exists but 404s until backfill regenerates).
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
bucket = sha256_hex[:3]
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
@@ -198,7 +216,7 @@ class GalleryService:
created_at=record.created_at,
effective_date=eff_date,
posted_at=posted_at,
thumbnail_url=thumbnail_url(record.sha256, record.mime),
thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
artist=artists.get(record.id),
)
for record, posted_at, eff_date in rows
@@ -306,7 +324,7 @@ class GalleryService:
"integrity_status": record.integrity_status,
"created_at": record.created_at.isoformat(),
"posted_at": posted_at.isoformat() if posted_at else None,
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
"thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
"artist": (
{"id": artist.id, "name": artist.name, "slug": artist.slug}
backend/app/services/post_feed_service.py
+4 -3
View File
@@ -207,6 +207,7 @@ class PostFeedService:
ImageRecord.primary_post_id,
ImageRecord.sha256,
ImageRecord.mime,
ImageRecord.thumbnail_path,
func.row_number().over(
partition_by=ImageRecord.primary_post_id,
order_by=ImageRecord.id.asc(),
@@ -220,18 +221,18 @@ class PostFeedService:
)
stmt = select(
ranked.c.id, ranked.c.primary_post_id,
ranked.c.sha256, ranked.c.mime, ranked.c.total,
ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total,
)
if limit is not None:
stmt = stmt.where(ranked.c.rn <= limit)
rows = (await self.session.execute(stmt)).all()
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
for img_id, pid, sha, mime, total in rows:
for img_id, pid, sha, mime, tp, total in rows:
entry = out.setdefault(pid, {"thumbs": [], "more": 0})
entry["thumbs"].append({
"image_id": img_id,
"thumbnail_url": thumbnail_url(sha, mime),
"thumbnail_url": thumbnail_url(tp, sha, mime),
"mime": mime,
})
# `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT.
backend/app/services/series_service.py
+2 -1
View File
@@ -63,6 +63,7 @@ class SeriesService:
ImageRecord.sha256,
ImageRecord.mime,
ImageRecord.path,
ImageRecord.thumbnail_path,
)
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
.where(SeriesPage.series_tag_id == series_tag_id)
@@ -75,7 +76,7 @@ class SeriesService:
{
"image_id": r.image_id,
"page_number": r.page_number,
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
"image_url": f"/images/{r.path.split('/images/', 1)[-1]}",
}
for r in rows
backend/app/services/showcase_service.py
+1 -1
View File
@@ -34,7 +34,7 @@ class ShowcaseService:
"mime": r.mime,
"width": r.width,
"height": r.height,
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
}
for r in rows
]
backend/app/services/tag_directory_service.py
+8 -3
View File
@@ -115,12 +115,17 @@ class TagDirectoryService:
.subquery()
)
stmt = (
select(sub.c.tag_id, ImageRecord.sha256, ImageRecord.mime)
select(
sub.c.tag_id,
ImageRecord.sha256,
ImageRecord.mime,
ImageRecord.thumbnail_path,
)
.join(ImageRecord, ImageRecord.id == sub.c.image_record_id)
.where(sub.c.rn <= 3)
.order_by(sub.c.tag_id, sub.c.rn)
)
out: dict[int, list[str]] = {}
for tag_id, sha, mime in (await self.session.execute(stmt)).all():
out.setdefault(tag_id, []).append(thumbnail_url(sha, mime))
for tag_id, sha, mime, tp in (await self.session.execute(stmt)).all():
out.setdefault(tag_id, []).append(thumbnail_url(tp, sha, mime))
return out
backend/app/utils/_archive_probe_runner.py
+33
View File
@@ -0,0 +1,33 @@
"""Subprocess entrypoint for safe_probe.probe_archive — see safe_probe.py.
probe_archive spawns this via subprocess (not multiprocessing.Process)
because Celery's prefork worker pool runs tasks in DAEMON processes and
Python's multiprocessing forbids daemon processes from spawning children
("AssertionError: daemonic processes are not allowed to have children",
operator-flagged 2026-05-30 — every archive import failed at task
startup). subprocess has no such restriction; we still get crash-
isolation because a probe segfault/OOM exits non-zero rather than
killing the worker.
Prints a single JSON line on stdout: {"status": "ok"|"error",
"detail": "..."?}. Exit code 0 for clean outcomes; non-zero exit
(signal / OOM-kill / unhandled exception) is the poison-pill signature
the parent maps to ProbeResult(crashed=True).
"""
import json
import sys
from .safe_probe import _run_probe
def main() -> int:
if len(sys.argv) != 2:
print(json.dumps({"status": "error", "detail": "usage: <path>"}))
return 2
status, detail = _run_probe(sys.argv[1])
print(json.dumps({"status": status, "detail": detail}))
return 0
if __name__ == "__main__":
sys.exit(main())
backend/app/utils/safe_probe.py
+54 -33
View File
@@ -28,8 +28,8 @@ Operator-requested 2026-05-28 (Layer 3).
"""
import json
import multiprocessing as mp
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
@@ -40,6 +40,13 @@ ARCHIVE_PROBE_TIMEOUT_SECONDS = 120
# art-pack archives while stopping a few-KB zip that expands to TB).
MAX_ARCHIVE_UNCOMPRESSED_BYTES = 4 * 1024 * 1024 * 1024
# Repo root for the subprocess cwd so `python -m backend.app.utils.*`
# resolves regardless of where Celery / pytest started. backend/app/utils
# = parents[0]; backend/app = parents[1]; backend = parents[2]; repo root
# = parents[3].
_REPO_ROOT = Path(__file__).resolve().parents[3]
_PROBE_RUNNER_MODULE = "backend.app.utils._archive_probe_runner"
@dataclass(frozen=True)
class ProbeResult:
@@ -91,54 +98,68 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) ->
def probe_archive(path: Path, *, timeout: float = ARCHIVE_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
"""Bomb-size guard + isolated integrity test for an archive."""
ctx = mp.get_context("spawn")
q = ctx.Queue()
proc = ctx.Process(target=_archive_probe_target, args=(str(path), q))
proc.start()
proc.join(timeout)
if proc.is_alive():
proc.terminate()
proc.join(5)
"""Bomb-size guard + isolated integrity test for an archive.
Runs via subprocess (not multiprocessing.Process) because Celery's
prefork worker pool is daemon-mode and Python's multiprocessing
forbids daemon processes from spawning children ("AssertionError:
daemonic processes are not allowed to have children"). subprocess
has no such restriction and still gives the crash isolation: a probe
segfault/OOM exits non-zero rather than killing the worker.
"""
try:
result = subprocess.run(
[sys.executable, "-m", _PROBE_RUNNER_MODULE, str(path)],
capture_output=True, text=True, timeout=timeout,
cwd=str(_REPO_ROOT),
)
except subprocess.TimeoutExpired:
return ProbeResult(ok=False, crashed=True, reason="archive probe timed out")
if proc.exitcode != 0:
# Negative exitcode = killed by signal (segfault); positive =
# the child os._exit'd or was OOM-killed. Either way the file
# hard-crashed the probe — the poison-pill signature.
if result.returncode != 0:
# Negative = killed by signal (segfault); positive = unhandled
# exception or OOM-kill. Either way: poison-pill signature.
return ProbeResult(
ok=False, crashed=True,
reason=f"archive probe crashed (exit {proc.exitcode})",
reason=f"archive probe crashed (exit {result.returncode})",
)
last_line = result.stdout.strip().splitlines()[-1:] or [""]
try:
outcome = q.get(timeout=5)
except Exception: # noqa: BLE001 — empty queue / broken pipe
return ProbeResult(ok=False, crashed=True, reason="archive probe produced no result")
status, detail = outcome
if status == "ok":
outcome = json.loads(last_line[0])
except json.JSONDecodeError as exc:
return ProbeResult(
ok=False, crashed=True,
reason=f"archive probe produced no parseable result: {exc}",
)
if outcome.get("status") == "ok":
return ProbeResult(ok=True)
return ProbeResult(ok=False, crashed=False, reason=detail)
return ProbeResult(
ok=False, crashed=False,
reason=outcome.get("detail") or "archive probe rejected",
)
def _archive_probe_target(path_str: str, q) -> None:
"""Runs in the spawned child. Reads member sizes (bomb guard) then
runs the format's integrity test. Puts ('ok', None) or
('error', reason). A crash/OOM here never reaches the queue — the
parent reads the non-zero exit code instead."""
def _run_probe(path_str: str) -> tuple[str, str | None]:
"""Pure-Python body of the archive probe — bomb-guard + integrity test.
Returns ('ok', None) or ('error', reason). Caught exceptions become
clean 'error' rejections; uncaught crashes in the subprocess become
non-zero exit codes (poison-pill signature) handled by probe_archive.
Exposed at the module level so the subprocess runner and tests both
call the same code path.
"""
path = Path(path_str)
ext = path.suffix.lower()
try:
total, test_bad = _inspect_archive(path, ext)
except Exception as exc: # noqa: BLE001 — clean rejection
q.put(("error", f"{type(exc).__name__}: {exc}"))
return
return ("error", f"{type(exc).__name__}: {exc}")
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
gib = total / (1024 ** 3)
q.put(("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap"))
return
return ("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap")
if test_bad is not None:
q.put(("error", f"integrity test failed at member {test_bad!r}"))
return
q.put(("ok", None))
return ("error", f"integrity test failed at member {test_bad!r}")
return ("ok", None)
def _inspect_archive(path: Path, ext: str):
frontend/src/stores/gallery.js
+15 -2
View File
@@ -2,6 +2,13 @@ import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import { useApi } from '../composables/useApi.js'
// Operator-confirmed 2026-05-30: fetch PAGE-sized chunks instead of one
// 50-item request so items render as each batch lands. Total initial
// count is unchanged (PAGE * INITIAL_BATCHES = 50). Infinite-scroll also
// pulls PAGE per trigger to keep appends progressive.
const PAGE = 5
const INITIAL_BATCHES = 10
export const useGalleryStore = defineStore('gallery', () => {
const api = useApi()
@@ -21,7 +28,13 @@ export const useGalleryStore = defineStore('gallery', () => {
images.value = []
dateGroups.value = []
nextCursor.value = null
await loadMore()
// Sequentially fetch INITIAL_BATCHES chunks so items render as each
// batch lands rather than blocking on one big response. Stop early
// when the backend reports no more pages.
for (let i = 0; i < INITIAL_BATCHES; i++) {
if (i > 0 && nextCursor.value === null) break
await loadMore()
}
}
async function loadMore() {
@@ -30,7 +43,7 @@ export const useGalleryStore = defineStore('gallery', () => {
error.value = null
const myId = ++inflightId
try {
const params = { limit: 50, ...activeFilterParam() }
const params = { limit: PAGE, ...activeFilterParam() }
if (nextCursor.value) params.cursor = nextCursor.value
const body = await api.get('/api/gallery/scroll', { params })
if (myId !== inflightId) return // stale response
frontend/src/stores/showcase.js
+20 -4
View File
@@ -3,7 +3,13 @@ import { ref, computed } from 'vue'
import { useApi } from '../composables/useApi.js'
import { useAsyncAction } from '../composables/useAsyncAction.js'
const PAGE = 60
// Operator-confirmed 2026-05-30: instead of one 60-item request, fetch
// PAGE-sized chunks sequentially so items render as each batch lands
// rather than blocking on the full 60-item response. Total initial count
// is unchanged (PAGE * INITIAL_BATCHES = 60). Infinite-scroll also pulls
// PAGE items per trigger so subsequent appends stay progressive too.
const PAGE = 5
const INITIAL_BATCHES = 12
export const useShowcaseStore = defineStore('showcase', () => {
const api = useApi()
@@ -23,11 +29,21 @@ export const useShowcaseStore = defineStore('showcase', () => {
})
}
async function shuffle() {
// Reset state and fetch INITIAL_BATCHES chunks in sequence. Used by
// mount, the Shuffle button, and the R-key handler — all want the
// same progressive-cascade behavior.
async function loadInitial() {
images.value = []
seen.clear()
exhausted.value = false
await fetchPage()
for (let i = 0; i < INITIAL_BATCHES; i++) {
if (exhausted.value) break
await fetchPage()
}
}
async function shuffle() {
await loadInitial()
}
const hasMore = computed(() => !exhausted.value)
@@ -35,5 +51,5 @@ export const useShowcaseStore = defineStore('showcase', () => {
() => !loading.value && images.value.length === 0 && error.value === null
)
return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle }
return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle, loadInitial }
})
frontend/src/views/ShowcaseView.vue
+1 -1
View File
@@ -70,7 +70,7 @@ function onKeydown(e) {
}
onMounted(() => {
if (store.images.length === 0) store.fetchPage()
if (store.images.length === 0) store.loadInitial()
window.addEventListener('keydown', onKeydown)
})
onUnmounted(() => window.removeEventListener('keydown', onKeydown))
tests/test_safe_probe.py
+9 -12
View File
@@ -1,12 +1,11 @@
"""Layer-3 subprocess-isolated probe tests.
The bomb-guard cap is exercised against `_archive_probe_target` directly
(in-process, where a monkeypatch on the module constant takes effect) —
spawn re-imports the module in the child, so a parent-process
monkeypatch wouldn't reach the spawned worker.
The bomb-guard cap is exercised against `_run_probe` directly (in-process,
where a monkeypatch on the module constant takes effect) — the real probe
runs in a subprocess that re-imports the module, so a parent-process
monkeypatch wouldn't reach the spawned interpreter.
"""
import multiprocessing as mp
import zipfile
from pathlib import Path
@@ -46,16 +45,14 @@ def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
assert bad is None
def test_archive_probe_target_bomb_guard(tmp_path, monkeypatch):
"""In-process call to the child target so the monkeypatched cap
takes effect. A normal zip whose uncompressed size exceeds the
(lowered) cap is rejected with the bomb-guard reason."""
def test_run_probe_bomb_guard(tmp_path, monkeypatch):
"""In-process call to _run_probe so the monkeypatched cap takes effect.
The public probe_archive runs in a subprocess which re-imports the
module and wouldn't see the patched constant."""
monkeypatch.setattr(safe_probe, "MAX_ARCHIVE_UNCOMPRESSED_BYTES", 10)
z = tmp_path / "bomb.zip"
_zip(z, {"big.txt": b"x" * 5000}) # 5000 uncompressed > 10-byte cap
q = mp.get_context("spawn").Queue()
safe_probe._archive_probe_target(str(z), q)
status, detail = q.get(timeout=5)
status, detail = safe_probe._run_probe(str(z))
assert status == "error"
assert "bomb-guard cap" in detail