diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 772fdc9..08ca44e 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -418,7 +418,8 @@ class DownloadService: # the duplicate file). Empty outside recapture mode. relink_pairs = getattr(dl_result, "relink_source_paths", None) or [] relinked = 0 - for rel_str, rel_url in relink_pairs: + post_linked = 0 + for rel_str, rel_url, rel_post_id in relink_pairs: rel_path = Path(rel_str) if not rel_path.exists(): # noqa: ASYNC240 continue @@ -428,13 +429,27 @@ class DownloadService: if await loop.run_in_executor(None, _relink): relinked += 1 + + # #1288: link the on-disk image to its Post. Recapture disk-skips the + # media (never re-imported), so a pre-existing image (e.g. one pulled + # under the old gallery-dl path) otherwise stays orphaned even after + # the post record is written. Idempotent for already-linked images. + def _link(p=rel_path, pid=rel_post_id): + return self.importer.link_existing_image_to_post( + p, pid, source=source_row, artist=artist, + ) + + if await loop.run_in_executor(None, _link): + post_linked += 1 if relink_pairs: # recapture diagnostic: how many on-disk images got their - # source_filehash backfilled (inline-image localization). < total is - # normal — files already carrying a filehash are skipped (NULL-only). + # source_filehash backfilled (inline-image localization) and how many + # got (re)linked to their Post. < total for source_filehash is normal + # (files already carrying a filehash are skipped, NULL-only). log.info( - "recap: relinked source_filehash on %d/%d on-disk image(s)", - relinked, len(relink_pairs), + "recap: relinked source_filehash on %d and linked %d/%d on-disk " + "image(s) to their post", + relinked, post_linked, len(relink_pairs), ) # Kick the off-platform file-host downloader for any links this run diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 748052e..71d13af 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -157,7 +157,7 @@ class DownloadResult: # pairs for already-present media whose ImageRecord.source_filehash should be # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty # on the gallery-dl path and outside recapture. - relink_source_paths: list[tuple[str, str]] = field(default_factory=list) + relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 2459836..69be5f8 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -1313,6 +1313,113 @@ class Importer: self.session.commit() return True + def link_existing_image_to_post( + self, path: Path, external_post_id: str, *, + source: Source | None = None, artist: Artist | None = None, + ) -> bool: + """Recapture back-link: associate an ALREADY-on-disk image (matched by + its stored path) with its post — the link _apply_sidecar normally makes + at per-media import time. + + Recapture disk-skips downloaded media, so a pre-existing image (e.g. one + pulled under the old gallery-dl path, imported as a bare record with no + post) never gets its `image_provenance` row / `primary_post_id`. This + backfills it from the walk's (on-disk path, post external id) pairing: + find the ImageRecord by path, find/attach the Post by (source, + external_post_id), upsert provenance. Idempotent (issue #1288). Returns + True when a record matched and was linked; no-op when the file has no + record or no id is given.""" + if not external_post_id: + return False + record = self.session.execute( + select(ImageRecord).where(ImageRecord.path == str(path)) + ).scalar_one_or_none() + if record is None: + return False + artist_id = record.artist_id or (artist.id if artist else None) + if artist_id is None: + return False + if record.artist_id is None: + record.artist_id = artist_id + post = self._find_or_create_post( + source_id=source.id if source else None, + external_post_id=str(external_post_id), + artist_id=artist_id, + ) + self._attach_provenance( + record, post, source_id=source.id if source else None, + ) + self.session.commit() + return True + + def _attach_provenance( + self, record: ImageRecord, post: Post, *, + source_id: int | None, captured_metadata: dict | None = None, + ) -> None: + """Upsert the (image, post) `image_provenance` link + keep + `primary_post_id` and the denormalized gallery sort keys aligned. Shared + by _apply_sidecar (fresh per-media import) and link_existing_image_to_post + (recapture back-link, #1288) so the two paths can't diverge. Idempotent. + + Race-safe (image_record_id, post_id) upsert — mirrors the + _find_or_create_source/post savepoint pattern. The plain + SELECT-then-INSERT pattern lost a race when two workers ran on the same + (image, post) pair (e.g. the 5-min recovery sweep re-enqueued a + still-running long import), planting duplicates that then broke + .scalar_one_or_none() on every later deep-scan rederive + (MultipleResultsFound). Alembic 0021's uq_image_provenance_image_post + UNIQUE makes this savepoint trip on collision.""" + exists = self.session.execute( + select(ImageProvenance.id).where( + ImageProvenance.image_record_id == record.id, + ImageProvenance.post_id == post.id, + ) + ).scalar_one_or_none() + if exists is None: + sp = self.session.begin_nested() + try: + self.session.add( + ImageProvenance( + image_record_id=record.id, + post_id=post.id, + source_id=source_id, + captured_metadata=captured_metadata, + ) + ) + self.session.flush() + sp.commit() + except IntegrityError: + sp.rollback() + if record.primary_post_id is None: + record.primary_post_id = post.id + # Keep the denormalized gallery sort key (alembic 0035) aligned with + # the primary post's publish date so /scroll orders off + # ix_image_record_effective_date instead of COALESCE-ing across the + # post join. Only override when THIS post is the primary AND carries + # a date; otherwise the column keeps its created_at-equivalent server + # default (matches the old COALESCE(post_date, created_at) fallback). + if record.primary_post_id == post.id and post.post_date is not None: + record.effective_date = post.post_date + # earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this + # image's provenance posts, not just the primary — so the gallery can + # sort by original publish rather than the download/repost the primary + # points at. Recompute from provenance whenever a dated post is linked; + # the provenance row for THIS post was committed above, so the MIN + # includes it. Leaves the created_at default when no linked post is dated. + if post.post_date is not None: + earliest = self.session.execute( + select(func.min(Post.post_date)) + .select_from(ImageProvenance) + .join(Post, Post.id == ImageProvenance.post_id) + .where( + ImageProvenance.image_record_id == record.id, + Post.post_date.is_not(None), + ) + ).scalar_one_or_none() + if earliest is not None: + record.earliest_post_date = earliest + self.session.flush() + def _apply_sidecar( self, record: ImageRecord, @@ -1386,65 +1493,12 @@ class Importer: if not self.post_first: self._apply_post_fields(post, sd) - # Race-safe (image_record_id, post_id) upsert — mirrors the - # _find_or_create_source/post savepoint pattern. The plain - # SELECT-then-INSERT pattern lost a race when two workers ran - # _apply_sidecar on the same (image, post) pair (e.g. the 5-min - # recovery sweep re-enqueued a still-running long import), planting - # duplicates that then broke .scalar_one_or_none() on every later - # deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the - # uq_image_provenance_image_post UNIQUE so this savepoint actually - # trips on collision. - exists = self.session.execute( - select(ImageProvenance.id).where( - ImageProvenance.image_record_id == record.id, - ImageProvenance.post_id == post.id, - ) - ).scalar_one_or_none() - if exists is None: - sp = self.session.begin_nested() - try: - self.session.add( - ImageProvenance( - image_record_id=record.id, - post_id=post.id, - source_id=src.id if src else None, - captured_metadata=sd.raw, - ) - ) - self.session.flush() - sp.commit() - except IntegrityError: - sp.rollback() - if record.primary_post_id is None: - record.primary_post_id = post.id - # Keep the denormalized gallery sort key (alembic 0035) aligned with - # the primary post's publish date so /scroll orders off - # ix_image_record_effective_date instead of COALESCE-ing across the - # post join. Only override when THIS post is the primary AND carries - # a date; otherwise the column keeps its created_at-equivalent server - # default (matches the old COALESCE(post_date, created_at) fallback). - if record.primary_post_id == post.id and post.post_date is not None: - record.effective_date = post.post_date - # earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this - # image's provenance posts, not just the primary — so the gallery can - # sort by original publish rather than the download/repost the primary - # points at. Recompute from provenance whenever a dated post is linked; - # the provenance row for THIS post was committed above, so the MIN - # includes it. Leaves the created_at default when no linked post is dated. - if post.post_date is not None: - earliest = self.session.execute( - select(func.min(Post.post_date)) - .select_from(ImageProvenance) - .join(Post, Post.id == ImageProvenance.post_id) - .where( - ImageProvenance.image_record_id == record.id, - Post.post_date.is_not(None), - ) - ).scalar_one_or_none() - if earliest is not None: - record.earliest_post_date = earliest - self.session.flush() + # Link the (image, post) provenance + keep primary_post_id / the gallery + # sort keys aligned — shared with the recapture back-link path (#1288). + self._attach_provenance( + record, post, source_id=src.id if src else None, + captured_metadata=sd.raw, + ) def _copy_to_library( self, source: Path, sha: str, attribution_path: Path diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 42deb77..9b5d391 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -179,10 +179,11 @@ class Ingester: written: list[str] = [] post_records: list[str] = [] quarantined_paths: list[str] = [] - # #830 recapture: (on-disk path, CDN source_url) pairs for already-present - # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT - # re-downloading or unlinking the file. Empty outside recapture mode. - relink: list[tuple[str, str]] = [] + # #830 recapture: (on-disk path, CDN source_url, post_id) triples for + # already-present media, so phase 3 can (a) backfill the ImageRecord's + # source_filehash and (b) link the on-disk image to its Post (#1288) — + # WITHOUT re-downloading or unlinking the file. Empty outside recapture. + relink: list[tuple[str, str, str]] = [] downloaded = 0 errors = 0 quarantined = 0 @@ -410,12 +411,15 @@ class Ingester: to_clear.append(key) skipped_count += 1 consecutive_seen += 1 - # #830 recapture: surface (on-disk path, CDN url) so phase - # 3 can backfill source_filehash for inline-image - # localization — a SEPARATE non-deleting channel, never the - # import list (which would unlink the file, per above). + # #830/#1288 recapture: surface (on-disk path, CDN url, + # post_id) so phase 3 can backfill source_filehash AND link + # the on-disk image to its Post — a SEPARATE non-deleting + # channel, never the import list (which would unlink the + # file, per above). if recapture and outcome.path is not None: - relink.append((str(outcome.path), media_item.url)) + relink.append( + (str(outcome.path), media_item.url, media_item.post_id) + ) elif outcome.status == "skipped_seen": skipped_count += 1 consecutive_seen += 1 diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index 7143663..1f4d01a 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -326,7 +326,10 @@ async def _current_heads(session: AsyncSession, embedding_version: str): { "tag_id": r.tag_id, "name": r.name, - "category": _CATEGORY.get(r.kind, "general"), + # System tags (wip/banner/editor) are kind=general but group under + # their OWN "system" suggestion category so the operator reviews + # them apart from content tags (they still train as general heads). + "category": "system" if r.is_system else _CATEGORY.get(r.kind, "general"), "auto_apply_threshold": r.auto_apply_threshold, "is_system": bool(r.is_system), } diff --git a/backend/app/services/pixiv_client.py b/backend/app/services/pixiv_client.py index 3635ca4..e6b5fea 100644 --- a/backend/app/services/pixiv_client.py +++ b/backend/app/services/pixiv_client.py @@ -435,24 +435,59 @@ class PixivClient: ) ] - def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]: - """The ugoira frame zip (gallery-dl's default non-original mode): - `/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080 - swap. Frame delays are memoized onto the work so the post record - captures them (a future ugoira→video conversion needs the timings — - the zip alone has none). A metadata failure downgrades to 'no media' - with a warning (matching gallery-dl) instead of failing the walk — - except auth failures, which stay loud.""" + def _ugoira_meta(self, work: dict, pid: str) -> dict | None: + """Fetch + memoize the ugoira metadata (frames + zip urls) for a work. + + Idempotent and cached on the work dict, so the post record and the + media extraction share ONE `/v1/ugoira/metadata` call regardless of + which runs first (the core writes the post record BEFORE it extracts + media). Returns None — and caches the miss — on a non-auth failure + (matching gallery-dl's downgrade); auth failures stay loud.""" + if "_ugoira_meta" in work: + return work["_ugoira_meta"] try: body = self._call("/v1/ugoira/metadata", {"illust_id": pid}) meta = body["ugoira_metadata"] - zip_url = meta["zip_urls"]["medium"] except PixivAuthError: raise except (PixivAPIError, KeyError, TypeError) as exc: log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc) - return [] + work["_ugoira_meta"] = None + return None + work["_ugoira_meta"] = meta + # Frame delays: a future ugoira→video conversion needs the timings (the + # zip alone has none), so the post record captures them. work["_ugoira_frames"] = meta.get("frames") or [] + return meta + + def fetch_ugoira_frames(self, post: dict) -> None: + """Populate `post['_work']['_ugoira_frames']` for an ugoira post (no-op + otherwise). The core writes the post record BEFORE extract_media, so + without this the frame timings would never reach the record; this + fetches (and memoizes, so extract_media reuses it) the metadata. Injected + into the downloader by the ingester, mirroring Patreon's content_fetcher. + Auth errors propagate; other failures leave frames unset.""" + work = post.get("_work") or {} + if work.get("type") != "ugoira": + return + pid = str(post.get("id") or "") + if pid: + self._ugoira_meta(work, pid) + + def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]: + """The ugoira frame zip (gallery-dl's default non-original mode): + `/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080 + swap. A metadata failure downgrades to 'no media' with a warning + (matching gallery-dl) instead of failing the walk — except auth + failures, which stay loud.""" + meta = self._ugoira_meta(work, pid) + if meta is None: + return [] + try: + zip_url = meta["zip_urls"]["medium"] + except (KeyError, TypeError) as exc: + log.warning("Pixiv ugoira zip url missing for %s: %s", pid, exc) + return [] url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1) return [ MediaItem( diff --git a/backend/app/services/pixiv_downloader.py b/backend/app/services/pixiv_downloader.py index 24e1226..532398c 100644 --- a/backend/app/services/pixiv_downloader.py +++ b/backend/app/services/pixiv_downloader.py @@ -102,11 +102,16 @@ class PixivDownloader(BaseNativeDownloader): validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, + ugoira_frames_fetcher: Callable[[dict], None] | None = None, ): super().__init__( images_root, cookies_path, platform="pixiv", validate=validate, rate_limit=rate_limit, session=session, ) + # Injected by the ingester (client.fetch_ugoira_frames) so write_post_record + # can populate frame timings — which extract_media memoizes, but the core + # writes the post record FIRST. Mirrors Patreon's content_fetcher. + self._ugoira_frames_fetcher = ugoira_frames_fetcher if session is None: # i.pximg.net 403s any GET without the app Referer; mirror the # client's full app-header profile (gallery-dl serves media off @@ -248,7 +253,16 @@ class PixivDownloader(BaseNativeDownloader): "account": user.get("account"), "name": user.get("name"), } - # Memoized by extract_media's ugoira branch; the zip has no timings. + # Ugoira frame timings. extract_media memoizes these, but the core writes + # the post record BEFORE extracting media, so fetch them here (shared + + # idempotent via the client's memoization) so the record actually keeps + # them — the zip carries no timings. + if ( + work.get("type") == "ugoira" + and not work.get("_ugoira_frames") + and self._ugoira_frames_fetcher is not None + ): + self._ugoira_frames_fetcher(post) frames = work.get("_ugoira_frames") if frames: data["ugoira_frames"] = frames diff --git a/backend/app/services/pixiv_ingester.py b/backend/app/services/pixiv_ingester.py index 663f292..8930de1 100644 --- a/backend/app/services/pixiv_ingester.py +++ b/backend/app/services/pixiv_ingester.py @@ -81,6 +81,10 @@ class PixivIngester(Ingester): if downloader is not None else PixivDownloader( self.images_root, cookies_path, validate=validate, rate_limit=rate_limit, + # write_post_record runs before extract_media in the core, so it + # fetches ugoira frame timings via the SAME client (shared, + # memoized) — else the record's ugoira_frames stays empty. + ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames, ) ) super().__init__( diff --git a/frontend/src/components/modal/SuggestionsPanel.vue b/frontend/src/components/modal/SuggestionsPanel.vue index 50779d5..79f7ba6 100644 --- a/frontend/src/components/modal/SuggestionsPanel.vue +++ b/frontend/src/components/modal/SuggestionsPanel.vue @@ -16,6 +16,16 @@ of the rail (ImageViewer side layout), so a long suggestion set no longer needs an internal scroll cap to keep Related reachable. -->
+ +