From 544e30bfb9ed8f28366d314057a42b783b218165 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Jul 2026 13:49:04 -0400 Subject: [PATCH] fix(pixiv): match gallery-dl's exact on-disk filename to avoid re-download at cutover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native downloader used the Windows-safe sanitize_segment, but gallery-dl on Linux (path-restrict auto→'/', path-remove default control chars, path-strip auto→'') replaces ONLY '/' and deletes control chars — the Windows-forbidden set (<>:"|?*) and trailing dots/spaces stay RAW in on-disk titles. Any pixiv title with those chars would therefore miss the tier-2 disk-skip and re-download the whole work at cutover (seen-ledger starts empty). Replace sanitize_segment with gdl_clean_filename, a byte-exact mirror of gallery-dl 1.32.5 build_filename (verified against path.py). Directory + template already matched; this closes the last parity gap. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/services/pixiv_downloader.py | 31 +++++++++++++++++++++--- tests/test_pixiv_downloader.py | 21 +++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/backend/app/services/pixiv_downloader.py b/backend/app/services/pixiv_downloader.py index 63bca1e..24e1226 100644 --- a/backend/app/services/pixiv_downloader.py +++ b/backend/app/services/pixiv_downloader.py @@ -34,6 +34,7 @@ from __future__ import annotations import json import logging +import re import time from collections.abc import Callable from pathlib import Path @@ -44,12 +45,32 @@ from .native_ingest_common import ( BaseNativeDownloader, MediaOutcome, PostRecordOutcome, - sanitize_segment, ) from .pixiv_client import PIXIV_APP_HEADERS, rating_label log = logging.getLogger(__name__) +# Control chars (0x00–0x1f + 0x7f DEL) — gallery-dl's default `path-remove`. +_GDL_PATH_REMOVE_RE = re.compile(r"[\x00-\x1f\x7f]") + + +def gdl_clean_filename(name: str) -> str: + """Reproduce gallery-dl's on-disk filename EXACTLY as it wrote it on this + Linux host, so the tier-2 disk-skip recognizes pre-cutover files instead of + re-downloading them. + + gallery-dl's PathFormat.build_filename is `clean_path(clean_segment(name))`. + On Linux (verified against gallery-dl 1.32.5 path.py) the defaults resolve to: + - path-restrict "auto" → "/" → clean_segment replaces ONLY "/" → "_" + - path-remove "\\x00-\\x1f\\x7f" → clean_path DELETES control chars + - path-strip "auto" → "" → NO trailing dot/space stripping + Crucially it does NOT touch the Windows-forbidden set (<>:"|?*) — those stay + raw in titles on disk. A stricter sanitizer here would rename any such title, + miss the on-disk match, and re-pull the whole work. Order mirrors gallery-dl + (segment inner, path outer); for these disjoint char sets it's commutative. + """ + return _GDL_PATH_REMOVE_RE.sub("", name.replace("/", "_")) + # Enrichment keys copied verbatim from the app-API work dict into the post # record (they're already JSON scalars/objects). Everything lands in # Post.raw_metadata via the importer, so the archive keeps pixiv's stats and @@ -150,9 +171,11 @@ class PixivDownloader(BaseNativeDownloader): if seen and not recapture: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) - # The client's filename already carries the {id}_{title50}_{NN} shape; - # only path-restrict sanitization happens here. - media_path = flat_dir / sanitize_segment(media.filename) + # The client's filename already carries the {id}_{title50}_{NN} shape + # (raw title, gallery-dl-template order); clean it to the byte-exact + # name gallery-dl wrote on disk so tier-2 disk-skip matches (else a + # re-download of the whole work). See gdl_clean_filename. + media_path = flat_dir / gdl_clean_filename(media.filename) if media_path.exists(): # tier-2: already on disk return MediaOutcome( diff --git a/tests/test_pixiv_downloader.py b/tests/test_pixiv_downloader.py index 636db77..ed45603 100644 --- a/tests/test_pixiv_downloader.py +++ b/tests/test_pixiv_downloader.py @@ -178,11 +178,15 @@ def test_download_post_honours_should_stop(tmp_path): assert outcomes == [] -def test_filename_is_sanitized(tmp_path): +def test_filename_matches_gallery_dl_linux_restrict(tmp_path): + # gallery-dl on Linux replaces ONLY "/" and deletes control chars — the + # Windows-forbidden set (<>:"|?*) stays RAW in the on-disk name. Matching + # that byte-for-byte is what lets the tier-2 disk-skip recognize + # pre-cutover files; a stricter sanitizer would re-download them. post, _ = _post_and_media(222) weird = MediaItem( url="https://i.pximg.net/img-original/img/2026/06/18/12/30/00/222_p0.jpg", - filename='222_What? A "Title"/Slash_00.jpg', + filename='222_What? A "Title"/Slash\x0900.jpg', # ? " stay, / → _, \t dropped kind="image", filehash=None, post_id="222", @@ -191,7 +195,18 @@ def test_filename_is_sanitized(tmp_path): dl = _downloader(tmp_path) outcomes = dl.download_post(post, [weird], "artist-a") assert outcomes[0].status == "downloaded" - assert outcomes[0].path.name == "222_What_ A _Title__Slash_00.jpg" + assert outcomes[0].path.name == '222_What? A "Title"_Slash00.jpg' + + +def test_gdl_clean_filename_parity(): + from backend.app.services.pixiv_downloader import gdl_clean_filename + # Only "/" is replaced; the Windows-forbidden set is untouched. + assert gdl_clean_filename('a<>:"|?*b.png') == 'a<>:"|?*b.png' + assert gdl_clean_filename("a/b/c.png") == "a_b_c.png" + # Control chars (newline, tab, DEL) are deleted, not replaced. + assert gdl_clean_filename("a\nb\tc\x7fd.png") == "abcd.png" + # Trailing dots/spaces are NOT stripped on Linux. + assert gdl_clean_filename("title . .png") == "title . .png" def test_own_session_carries_app_referer(tmp_path):