Merge pull request 'Recapture image→post linking (#1288) + ugoira timings + system suggestion category' (#192) from dev into main
Build images / sign-extension (push) Successful in 3s
CI / lint (push) Successful in 3s
Build images / build-agent (push) Successful in 7s
Build images / build-ml (push) Successful in 9s
Build images / build-web (push) Successful in 10s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 41s
CI / integration (push) Successful in 3m36s

This commit was merged in pull request #192.
This commit is contained in:
2026-07-04 20:36:58 -04:00
16 changed files with 362 additions and 94 deletions
+20 -5
View File
@@ -418,7 +418,8 @@ class DownloadService:
# the duplicate file). Empty outside recapture mode. # the duplicate file). Empty outside recapture mode.
relink_pairs = getattr(dl_result, "relink_source_paths", None) or [] relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
relinked = 0 relinked = 0
for rel_str, rel_url in relink_pairs: post_linked = 0
for rel_str, rel_url, rel_post_id in relink_pairs:
rel_path = Path(rel_str) rel_path = Path(rel_str)
if not rel_path.exists(): # noqa: ASYNC240 if not rel_path.exists(): # noqa: ASYNC240
continue continue
@@ -428,13 +429,27 @@ class DownloadService:
if await loop.run_in_executor(None, _relink): if await loop.run_in_executor(None, _relink):
relinked += 1 relinked += 1
# #1288: link the on-disk image to its Post. Recapture disk-skips the
# media (never re-imported), so a pre-existing image (e.g. one pulled
# under the old gallery-dl path) otherwise stays orphaned even after
# the post record is written. Idempotent for already-linked images.
def _link(p=rel_path, pid=rel_post_id):
return self.importer.link_existing_image_to_post(
p, pid, source=source_row, artist=artist,
)
if await loop.run_in_executor(None, _link):
post_linked += 1
if relink_pairs: if relink_pairs:
# recapture diagnostic: how many on-disk images got their # recapture diagnostic: how many on-disk images got their
# source_filehash backfilled (inline-image localization). < total is # source_filehash backfilled (inline-image localization) and how many
# normal — files already carrying a filehash are skipped (NULL-only). # got (re)linked to their Post. < total for source_filehash is normal
# (files already carrying a filehash are skipped, NULL-only).
log.info( log.info(
"recap: relinked source_filehash on %d/%d on-disk image(s)", "recap: relinked source_filehash on %d and linked %d/%d on-disk "
relinked, len(relink_pairs), "image(s) to their post",
relinked, post_linked, len(relink_pairs),
) )
# Kick the off-platform file-host downloader for any links this run # Kick the off-platform file-host downloader for any links this run
+1 -1
View File
@@ -157,7 +157,7 @@ class DownloadResult:
# pairs for already-present media whose ImageRecord.source_filehash should be # pairs for already-present media whose ImageRecord.source_filehash should be
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
# on the gallery-dl path and outside recapture. # on the gallery-dl path and outside recapture.
relink_source_paths: list[tuple[str, str]] = field(default_factory=list) relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
stdout: str = "" stdout: str = ""
stderr: str = "" stderr: str = ""
return_code: int = 0 return_code: int = 0
+113 -59
View File
@@ -1313,6 +1313,113 @@ class Importer:
self.session.commit() self.session.commit()
return True return True
def link_existing_image_to_post(
self, path: Path, external_post_id: str, *,
source: Source | None = None, artist: Artist | None = None,
) -> bool:
"""Recapture back-link: associate an ALREADY-on-disk image (matched by
its stored path) with its post — the link _apply_sidecar normally makes
at per-media import time.
Recapture disk-skips downloaded media, so a pre-existing image (e.g. one
pulled under the old gallery-dl path, imported as a bare record with no
post) never gets its `image_provenance` row / `primary_post_id`. This
backfills it from the walk's (on-disk path, post external id) pairing:
find the ImageRecord by path, find/attach the Post by (source,
external_post_id), upsert provenance. Idempotent (issue #1288). Returns
True when a record matched and was linked; no-op when the file has no
record or no id is given."""
if not external_post_id:
return False
record = self.session.execute(
select(ImageRecord).where(ImageRecord.path == str(path))
).scalar_one_or_none()
if record is None:
return False
artist_id = record.artist_id or (artist.id if artist else None)
if artist_id is None:
return False
if record.artist_id is None:
record.artist_id = artist_id
post = self._find_or_create_post(
source_id=source.id if source else None,
external_post_id=str(external_post_id),
artist_id=artist_id,
)
self._attach_provenance(
record, post, source_id=source.id if source else None,
)
self.session.commit()
return True
def _attach_provenance(
self, record: ImageRecord, post: Post, *,
source_id: int | None, captured_metadata: dict | None = None,
) -> None:
"""Upsert the (image, post) `image_provenance` link + keep
`primary_post_id` and the denormalized gallery sort keys aligned. Shared
by _apply_sidecar (fresh per-media import) and link_existing_image_to_post
(recapture back-link, #1288) so the two paths can't diverge. Idempotent.
Race-safe (image_record_id, post_id) upsert — mirrors the
_find_or_create_source/post savepoint pattern. The plain
SELECT-then-INSERT pattern lost a race when two workers ran on the same
(image, post) pair (e.g. the 5-min recovery sweep re-enqueued a
still-running long import), planting duplicates that then broke
.scalar_one_or_none() on every later deep-scan rederive
(MultipleResultsFound). Alembic 0021's uq_image_provenance_image_post
UNIQUE makes this savepoint trip on collision."""
exists = self.session.execute(
select(ImageProvenance.id).where(
ImageProvenance.image_record_id == record.id,
ImageProvenance.post_id == post.id,
)
).scalar_one_or_none()
if exists is None:
sp = self.session.begin_nested()
try:
self.session.add(
ImageProvenance(
image_record_id=record.id,
post_id=post.id,
source_id=source_id,
captured_metadata=captured_metadata,
)
)
self.session.flush()
sp.commit()
except IntegrityError:
sp.rollback()
if record.primary_post_id is None:
record.primary_post_id = post.id
# Keep the denormalized gallery sort key (alembic 0035) aligned with
# the primary post's publish date so /scroll orders off
# ix_image_record_effective_date instead of COALESCE-ing across the
# post join. Only override when THIS post is the primary AND carries
# a date; otherwise the column keeps its created_at-equivalent server
# default (matches the old COALESCE(post_date, created_at) fallback).
if record.primary_post_id == post.id and post.post_date is not None:
record.effective_date = post.post_date
# earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
# image's provenance posts, not just the primary — so the gallery can
# sort by original publish rather than the download/repost the primary
# points at. Recompute from provenance whenever a dated post is linked;
# the provenance row for THIS post was committed above, so the MIN
# includes it. Leaves the created_at default when no linked post is dated.
if post.post_date is not None:
earliest = self.session.execute(
select(func.min(Post.post_date))
.select_from(ImageProvenance)
.join(Post, Post.id == ImageProvenance.post_id)
.where(
ImageProvenance.image_record_id == record.id,
Post.post_date.is_not(None),
)
).scalar_one_or_none()
if earliest is not None:
record.earliest_post_date = earliest
self.session.flush()
def _apply_sidecar( def _apply_sidecar(
self, self,
record: ImageRecord, record: ImageRecord,
@@ -1386,65 +1493,12 @@ class Importer:
if not self.post_first: if not self.post_first:
self._apply_post_fields(post, sd) self._apply_post_fields(post, sd)
# Race-safe (image_record_id, post_id) upsert — mirrors the # Link the (image, post) provenance + keep primary_post_id / the gallery
# _find_or_create_source/post savepoint pattern. The plain # sort keys aligned — shared with the recapture back-link path (#1288).
# SELECT-then-INSERT pattern lost a race when two workers ran self._attach_provenance(
# _apply_sidecar on the same (image, post) pair (e.g. the 5-min record, post, source_id=src.id if src else None,
# recovery sweep re-enqueued a still-running long import), planting captured_metadata=sd.raw,
# duplicates that then broke .scalar_one_or_none() on every later )
# deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
# uq_image_provenance_image_post UNIQUE so this savepoint actually
# trips on collision.
exists = self.session.execute(
select(ImageProvenance.id).where(
ImageProvenance.image_record_id == record.id,
ImageProvenance.post_id == post.id,
)
).scalar_one_or_none()
if exists is None:
sp = self.session.begin_nested()
try:
self.session.add(
ImageProvenance(
image_record_id=record.id,
post_id=post.id,
source_id=src.id if src else None,
captured_metadata=sd.raw,
)
)
self.session.flush()
sp.commit()
except IntegrityError:
sp.rollback()
if record.primary_post_id is None:
record.primary_post_id = post.id
# Keep the denormalized gallery sort key (alembic 0035) aligned with
# the primary post's publish date so /scroll orders off
# ix_image_record_effective_date instead of COALESCE-ing across the
# post join. Only override when THIS post is the primary AND carries
# a date; otherwise the column keeps its created_at-equivalent server
# default (matches the old COALESCE(post_date, created_at) fallback).
if record.primary_post_id == post.id and post.post_date is not None:
record.effective_date = post.post_date
# earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
# image's provenance posts, not just the primary — so the gallery can
# sort by original publish rather than the download/repost the primary
# points at. Recompute from provenance whenever a dated post is linked;
# the provenance row for THIS post was committed above, so the MIN
# includes it. Leaves the created_at default when no linked post is dated.
if post.post_date is not None:
earliest = self.session.execute(
select(func.min(Post.post_date))
.select_from(ImageProvenance)
.join(Post, Post.id == ImageProvenance.post_id)
.where(
ImageProvenance.image_record_id == record.id,
Post.post_date.is_not(None),
)
).scalar_one_or_none()
if earliest is not None:
record.earliest_post_date = earliest
self.session.flush()
def _copy_to_library( def _copy_to_library(
self, source: Path, sha: str, attribution_path: Path self, source: Path, sha: str, attribution_path: Path
+13 -9
View File
@@ -179,10 +179,11 @@ class Ingester:
written: list[str] = [] written: list[str] = []
post_records: list[str] = [] post_records: list[str] = []
quarantined_paths: list[str] = [] quarantined_paths: list[str] = []
# #830 recapture: (on-disk path, CDN source_url) pairs for already-present # #830 recapture: (on-disk path, CDN source_url, post_id) triples for
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT # already-present media, so phase 3 can (a) backfill the ImageRecord's
# re-downloading or unlinking the file. Empty outside recapture mode. # source_filehash and (b) link the on-disk image to its Post (#1288) —
relink: list[tuple[str, str]] = [] # WITHOUT re-downloading or unlinking the file. Empty outside recapture.
relink: list[tuple[str, str, str]] = []
downloaded = 0 downloaded = 0
errors = 0 errors = 0
quarantined = 0 quarantined = 0
@@ -410,12 +411,15 @@ class Ingester:
to_clear.append(key) to_clear.append(key)
skipped_count += 1 skipped_count += 1
consecutive_seen += 1 consecutive_seen += 1
# #830 recapture: surface (on-disk path, CDN url) so phase # #830/#1288 recapture: surface (on-disk path, CDN url,
# 3 can backfill source_filehash for inline-image # post_id) so phase 3 can backfill source_filehash AND link
# localization — a SEPARATE non-deleting channel, never the # the on-disk image to its Post — a SEPARATE non-deleting
# import list (which would unlink the file, per above). # channel, never the import list (which would unlink the
# file, per above).
if recapture and outcome.path is not None: if recapture and outcome.path is not None:
relink.append((str(outcome.path), media_item.url)) relink.append(
(str(outcome.path), media_item.url, media_item.post_id)
)
elif outcome.status == "skipped_seen": elif outcome.status == "skipped_seen":
skipped_count += 1 skipped_count += 1
consecutive_seen += 1 consecutive_seen += 1
+4 -1
View File
@@ -326,7 +326,10 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
{ {
"tag_id": r.tag_id, "tag_id": r.tag_id,
"name": r.name, "name": r.name,
"category": _CATEGORY.get(r.kind, "general"), # System tags (wip/banner/editor) are kind=general but group under
# their OWN "system" suggestion category so the operator reviews
# them apart from content tags (they still train as general heads).
"category": "system" if r.is_system else _CATEGORY.get(r.kind, "general"),
"auto_apply_threshold": r.auto_apply_threshold, "auto_apply_threshold": r.auto_apply_threshold,
"is_system": bool(r.is_system), "is_system": bool(r.is_system),
} }
+45 -10
View File
@@ -435,24 +435,59 @@ class PixivClient:
) )
] ]
def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]: def _ugoira_meta(self, work: dict, pid: str) -> dict | None:
"""The ugoira frame zip (gallery-dl's default non-original mode): """Fetch + memoize the ugoira metadata (frames + zip urls) for a work.
`/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080
swap. Frame delays are memoized onto the work so the post record Idempotent and cached on the work dict, so the post record and the
captures them (a future ugoira→video conversion needs the timings — media extraction share ONE `/v1/ugoira/metadata` call regardless of
the zip alone has none). A metadata failure downgrades to 'no media' which runs first (the core writes the post record BEFORE it extracts
with a warning (matching gallery-dl) instead of failing the walk — media). Returns None — and caches the miss — on a non-auth failure
except auth failures, which stay loud.""" (matching gallery-dl's downgrade); auth failures stay loud."""
if "_ugoira_meta" in work:
return work["_ugoira_meta"]
try: try:
body = self._call("/v1/ugoira/metadata", {"illust_id": pid}) body = self._call("/v1/ugoira/metadata", {"illust_id": pid})
meta = body["ugoira_metadata"] meta = body["ugoira_metadata"]
zip_url = meta["zip_urls"]["medium"]
except PixivAuthError: except PixivAuthError:
raise raise
except (PixivAPIError, KeyError, TypeError) as exc: except (PixivAPIError, KeyError, TypeError) as exc:
log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc) log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc)
return [] work["_ugoira_meta"] = None
return None
work["_ugoira_meta"] = meta
# Frame delays: a future ugoira→video conversion needs the timings (the
# zip alone has none), so the post record captures them.
work["_ugoira_frames"] = meta.get("frames") or [] work["_ugoira_frames"] = meta.get("frames") or []
return meta
def fetch_ugoira_frames(self, post: dict) -> None:
"""Populate `post['_work']['_ugoira_frames']` for an ugoira post (no-op
otherwise). The core writes the post record BEFORE extract_media, so
without this the frame timings would never reach the record; this
fetches (and memoizes, so extract_media reuses it) the metadata. Injected
into the downloader by the ingester, mirroring Patreon's content_fetcher.
Auth errors propagate; other failures leave frames unset."""
work = post.get("_work") or {}
if work.get("type") != "ugoira":
return
pid = str(post.get("id") or "")
if pid:
self._ugoira_meta(work, pid)
def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]:
"""The ugoira frame zip (gallery-dl's default non-original mode):
`/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080
swap. A metadata failure downgrades to 'no media' with a warning
(matching gallery-dl) instead of failing the walk — except auth
failures, which stay loud."""
meta = self._ugoira_meta(work, pid)
if meta is None:
return []
try:
zip_url = meta["zip_urls"]["medium"]
except (KeyError, TypeError) as exc:
log.warning("Pixiv ugoira zip url missing for %s: %s", pid, exc)
return []
url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1) url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1)
return [ return [
MediaItem( MediaItem(
+15 -1
View File
@@ -102,11 +102,16 @@ class PixivDownloader(BaseNativeDownloader):
validate: bool = True, validate: bool = True,
rate_limit: float = 0.0, rate_limit: float = 0.0,
session: requests.Session | None = None, session: requests.Session | None = None,
ugoira_frames_fetcher: Callable[[dict], None] | None = None,
): ):
super().__init__( super().__init__(
images_root, cookies_path, platform="pixiv", images_root, cookies_path, platform="pixiv",
validate=validate, rate_limit=rate_limit, session=session, validate=validate, rate_limit=rate_limit, session=session,
) )
# Injected by the ingester (client.fetch_ugoira_frames) so write_post_record
# can populate frame timings — which extract_media memoizes, but the core
# writes the post record FIRST. Mirrors Patreon's content_fetcher.
self._ugoira_frames_fetcher = ugoira_frames_fetcher
if session is None: if session is None:
# i.pximg.net 403s any GET without the app Referer; mirror the # i.pximg.net 403s any GET without the app Referer; mirror the
# client's full app-header profile (gallery-dl serves media off # client's full app-header profile (gallery-dl serves media off
@@ -248,7 +253,16 @@ class PixivDownloader(BaseNativeDownloader):
"account": user.get("account"), "account": user.get("account"),
"name": user.get("name"), "name": user.get("name"),
} }
# Memoized by extract_media's ugoira branch; the zip has no timings. # Ugoira frame timings. extract_media memoizes these, but the core writes
# the post record BEFORE extracting media, so fetch them here (shared +
# idempotent via the client's memoization) so the record actually keeps
# them — the zip carries no timings.
if (
work.get("type") == "ugoira"
and not work.get("_ugoira_frames")
and self._ugoira_frames_fetcher is not None
):
self._ugoira_frames_fetcher(post)
frames = work.get("_ugoira_frames") frames = work.get("_ugoira_frames")
if frames: if frames:
data["ugoira_frames"] = frames data["ugoira_frames"] = frames
+4
View File
@@ -81,6 +81,10 @@ class PixivIngester(Ingester):
if downloader is not None if downloader is not None
else PixivDownloader( else PixivDownloader(
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit, self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
# write_post_record runs before extract_media in the core, so it
# fetches ugoira frame timings via the SAME client (shared,
# memoized) — else the record's ugoira_frames stays empty.
ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames,
) )
) )
super().__init__( super().__init__(
@@ -16,6 +16,16 @@
of the rail (ImageViewer side layout), so a long suggestion set no of the rail (ImageViewer side layout), so a long suggestion set no
longer needs an internal scroll cap to keep Related reachable. --> longer needs an internal scroll cap to keep Related reachable. -->
<div v-else class="fc-suggestions__list"> <div v-else class="fc-suggestions__list">
<!-- System hygiene tags (wip/banner/editor) surface in their own group,
first, so false positives are easy to spot and reject (reject-to-train
for these heads). Small set collapsible, open by default. -->
<SuggestionsCategoryGroup
v-if="store.byCategory.system && store.byCategory.system.length"
label="System" :items="store.byCategory.system"
collapsible :default-open="true"
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
@dismiss="onDismiss" @undismiss="onUndismiss"
/>
<SuggestionsCategoryGroup <SuggestionsCategoryGroup
v-for="cat in peopleCats" :key="cat" v-for="cat in peopleCats" :key="cat"
v-show="store.byCategory[cat] && store.byCategory[cat].length" v-show="store.byCategory[cat] && store.byCategory[cat].length"
@@ -154,6 +154,9 @@ const KNOWN_KINDS = new Set([
const KIND_ICONS = { const KIND_ICONS = {
general: 'mdi-tag', character: 'mdi-account-circle', general: 'mdi-tag', character: 'mdi-account-circle',
fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf', fandom: 'mdi-book-open-page-variant', series: 'mdi-bookshelf',
// 'system' is a suggestion category (wip/banner/editor), not a tag kind —
// matches the shield marker on system-tag chips.
system: 'mdi-shield-outline',
} }
function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' } function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
+6 -4
View File
@@ -5,11 +5,13 @@ import { useApi } from '../composables/useApi.js'
import { useAsyncAction } from '../composables/useAsyncAction.js' import { useAsyncAction } from '../composables/useAsyncAction.js'
import { useInflightToken } from '../composables/useInflightToken.js' import { useInflightToken } from '../composables/useInflightToken.js'
// Category display order: people first, general last. // Category display order: system (hygiene flags) first for quick review, then
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired — only // people, general last. 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01)
// character and general surface as suggestion categories now. // retired. 'system' groups the wip/banner/editor hygiene tags apart from content
export const CATEGORY_ORDER = ['character', 'general'] // tags (they're kind=general on the backend but surface under their own category).
export const CATEGORY_ORDER = ['system', 'character', 'general']
export const CATEGORY_LABELS = { export const CATEGORY_LABELS = {
system: 'System',
character: 'Character', character: 'Character',
general: 'General' general: 'General'
} }
+4 -2
View File
@@ -124,9 +124,11 @@ async def test_system_tag_surfaces_at_flat_floor_despite_high_threshold(db):
await _head(db, tag.id, slot=0, suggest_threshold=0.9) await _head(db, tag.id, slot=0, suggest_threshold=0.9)
await db.commit() await db.commit()
sl = await SuggestionService(db).for_image(img.id) sl = await SuggestionService(db).for_image(img.id)
# System tags surface under their OWN "system" category, not "general".
assert any( assert any(
s.canonical_tag_id == tag.id for s in sl.by_category.get("general", []) s.canonical_tag_id == tag.id for s in sl.by_category.get("system", [])
) )
assert not sl.by_category.get("general")
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -139,7 +141,7 @@ async def test_system_tag_below_floor_stays_hidden(db):
await _head(db, tag.id, slot=0, suggest_threshold=0.9) await _head(db, tag.id, slot=0, suggest_threshold=0.9)
await db.commit() await db.commit()
sl = await SuggestionService(db).for_image(img.id) sl = await SuggestionService(db).for_image(img.id)
assert not sl.by_category.get("general") assert not sl.by_category.get("system")
@pytest.mark.asyncio @pytest.mark.asyncio
+4 -2
View File
@@ -667,10 +667,12 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
assert downloader2.post_records == 1 assert downloader2.post_records == 1
assert res_recap.files_downloaded == 0 # no media re-download assert res_recap.files_downloaded == 0 # no media re-download
assert downloader2.download_calls == 0 assert downloader2.download_calls == 0
# The on-disk media is surfaced as a (path, CDN url) relink pair. # The on-disk media is surfaced as a (path, CDN url, post_id) relink triple
# (post_id drives the #1288 image→post back-link in phase 3).
assert len(res_recap.relink_source_paths) == 1 assert len(res_recap.relink_source_paths) == 1
rel_path, rel_url = res_recap.relink_source_paths[0] rel_path, rel_url, rel_post_id = res_recap.relink_source_paths[0]
assert rel_url == m1.url assert rel_url == m1.url
assert rel_post_id == m1.post_id
# Diagnostic summary surfaces the post-record + relink counts (operator reads # Diagnostic summary surfaces the post-record + relink counts (operator reads
# this off the event stdout to see what a recapture actually did). # this off the event stdout to see what a recapture actually did).
assert "1 post-record(s), 1 relinked" in res_recap.stdout assert "1 post-record(s), 1 relinked" in res_recap.stdout
+32
View File
@@ -200,6 +200,38 @@ def test_extract_media_ugoira_zip_swap(client, page1, monkeypatch):
assert post["_work"]["_ugoira_frames"] == frames assert post["_work"]["_ugoira_frames"] == frames
def test_fetch_ugoira_frames_memoizes_and_shares_one_call(client, page1, monkeypatch):
# fetch_ugoira_frames (called by write_post_record, which the core runs
# BEFORE extract_media) populates frames; extract_media then REUSES the
# memoized metadata — exactly one /v1/ugoira/metadata call total.
frames = [{"file": "000000.jpg", "delay": 90}]
calls = []
def fake_call(endpoint, params):
calls.append(endpoint)
return {"ugoira_metadata": {
"zip_urls": {"medium": (
"https://i.pximg.net/img-zip-ugoira/x/333_ugoira600x600.zip"
)},
"frames": frames,
}}
monkeypatch.setattr(client, "_call", fake_call)
post = _post_for(client, page1, 333)
client.fetch_ugoira_frames(post)
assert post["_work"]["_ugoira_frames"] == frames
items = client.extract_media(post, {}) # reuses memoized meta
assert items[0].media_id == "ugoira"
assert calls == ["/v1/ugoira/metadata"] # ONE fetch, not two
def test_fetch_ugoira_frames_noop_for_non_ugoira(client, page1, monkeypatch):
monkeypatch.setattr(client, "_call", lambda e, p: pytest.fail("should not fetch"))
post = _post_for(client, page1, 222) # a single-page illust
client.fetch_ugoira_frames(post)
assert "_ugoira_frames" not in post["_work"]
def test_extract_media_ugoira_metadata_failure_downgrades( def test_extract_media_ugoira_metadata_failure_downgrades(
client, page1, monkeypatch client, page1, monkeypatch
): ):
+34
View File
@@ -268,6 +268,40 @@ def test_write_post_record_includes_ugoira_frames(tmp_path):
assert data["ugoira_frames"] == frames assert data["ugoira_frames"] == frames
def test_write_post_record_fetches_ugoira_frames_when_absent(tmp_path):
# The core writes the post record BEFORE extract_media, so frames aren't
# memoized yet — the injected fetcher must populate them so the record keeps
# the timings (regression: they were silently always empty).
post = _post_only(333)
assert "_ugoira_frames" not in post["_work"]
frames = [{"file": "000000.jpg", "delay": 120}]
calls = []
def fetcher(p):
calls.append(p)
p["_work"]["_ugoira_frames"] = frames
dl = PixivDownloader(
tmp_path, validate=False, session=_FakeSession(),
ugoira_frames_fetcher=fetcher,
)
outcome = dl.write_post_record(post, "artist-a")
data = json.loads(outcome.path.read_text())
assert data["ugoira_frames"] == frames
assert len(calls) == 1
def test_write_post_record_non_ugoira_does_not_call_fetcher(tmp_path):
post = _post_only(111) # a plain multi-page illust
calls = []
dl = PixivDownloader(
tmp_path, validate=False, session=_FakeSession(),
ugoira_frames_fetcher=lambda p: calls.append(p),
)
dl.write_post_record(post, "artist-a")
assert calls == []
def test_write_post_record_without_id(tmp_path): def test_write_post_record_without_id(tmp_path):
dl = _downloader(tmp_path) dl = _downloader(tmp_path)
outcome = dl.write_post_record({"id": None, "attributes": {}}, "artist-a") outcome = dl.write_post_record({"id": None, "attributes": {}}, "artist-a")
+54
View File
@@ -158,6 +158,60 @@ def test_relink_source_filehash_no_match_is_noop(importer, import_layout):
) is False ) is False
def test_link_existing_image_to_post_backfills_provenance(importer, import_layout, db_sync):
"""#1288: an on-disk image imported BARE (no sidecar → no post link, as the
pre-cutover gallery-dl pixiv images were) gets linked to its post from the
recapture walk's (path, external_post_id) pairing — image_provenance +
primary_post_id — without re-importing. Idempotent."""
import_root, _ = import_layout
m = import_root / "Alice" / "img.jpg"
_split(m, "v")
rec = importer.session.get(ImageRecord, importer.import_one(m).image_id)
assert rec.primary_post_id is None # bare: no post link yet
artist = db_sync.get(Artist, rec.artist_id)
src = Source(
artist_id=artist.id, platform="pixiv",
url="https://www.pixiv.net/users/1", enabled=True,
)
db_sync.add(src)
db_sync.flush()
# The post already exists (upsert_post_record wrote it earlier in recapture).
post = Post(
source_id=src.id, artist_id=artist.id,
external_post_id="146132304", post_title="t",
)
db_sync.add(post)
db_sync.commit()
assert importer.link_existing_image_to_post(
Path(rec.path), "146132304", source=src, artist=artist,
) is True
importer.session.refresh(rec)
assert rec.primary_post_id == post.id
prov = db_sync.execute(select(ImageProvenance).where(
ImageProvenance.image_record_id == rec.id,
ImageProvenance.post_id == post.id,
)).scalar_one()
assert prov.source_id == src.id
# Idempotent: a second call re-affirms without duplicating the row.
assert importer.link_existing_image_to_post(
Path(rec.path), "146132304", source=src, artist=artist,
) is True
n = db_sync.execute(select(func.count(ImageProvenance.id)).where(
ImageProvenance.image_record_id == rec.id,
ImageProvenance.post_id == post.id,
)).scalar_one()
assert n == 1
def test_link_existing_image_to_post_no_match_is_noop(importer):
"""A path with no ImageRecord, or an empty external id, is a clean no-op."""
assert importer.link_existing_image_to_post(
Path("/nonexistent/x.jpg"), "999",
) is False
def test_reimport_same_post_idempotent(importer, import_layout): def test_reimport_same_post_idempotent(importer, import_layout):
import_root, _ = import_layout import_root, _ = import_layout
# Threshold 0: only an exact phash match collapses. Orthogonal splits # Threshold 0: only an exact phash match collapses. Orthogonal splits