fix(pixiv): match gallery-dl's exact on-disk filename to avoid re-download at cutover
The native downloader used the Windows-safe sanitize_segment, but gallery-dl on Linux (path-restrict auto→'/', path-remove default control chars, path-strip auto→'') replaces ONLY '/' and deletes control chars — the Windows-forbidden set (<>:"|?*) and trailing dots/spaces stay RAW in on-disk titles. Any pixiv title with those chars would therefore miss the tier-2 disk-skip and re-download the whole work at cutover (seen-ledger starts empty). Replace sanitize_segment with gdl_clean_filename, a byte-exact mirror of gallery-dl 1.32.5 build_filename (verified against path.py). Directory + template already matched; this closes the last parity gap. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -34,6 +34,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
@@ -44,12 +45,32 @@ from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
sanitize_segment,
|
||||
)
|
||||
from .pixiv_client import PIXIV_APP_HEADERS, rating_label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Control chars (0x00–0x1f + 0x7f DEL) — gallery-dl's default `path-remove`.
|
||||
_GDL_PATH_REMOVE_RE = re.compile(r"[\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
def gdl_clean_filename(name: str) -> str:
|
||||
"""Reproduce gallery-dl's on-disk filename EXACTLY as it wrote it on this
|
||||
Linux host, so the tier-2 disk-skip recognizes pre-cutover files instead of
|
||||
re-downloading them.
|
||||
|
||||
gallery-dl's PathFormat.build_filename is `clean_path(clean_segment(name))`.
|
||||
On Linux (verified against gallery-dl 1.32.5 path.py) the defaults resolve to:
|
||||
- path-restrict "auto" → "/" → clean_segment replaces ONLY "/" → "_"
|
||||
- path-remove "\\x00-\\x1f\\x7f" → clean_path DELETES control chars
|
||||
- path-strip "auto" → "" → NO trailing dot/space stripping
|
||||
Crucially it does NOT touch the Windows-forbidden set (<>:"|?*) — those stay
|
||||
raw in titles on disk. A stricter sanitizer here would rename any such title,
|
||||
miss the on-disk match, and re-pull the whole work. Order mirrors gallery-dl
|
||||
(segment inner, path outer); for these disjoint char sets it's commutative.
|
||||
"""
|
||||
return _GDL_PATH_REMOVE_RE.sub("", name.replace("/", "_"))
|
||||
|
||||
# Enrichment keys copied verbatim from the app-API work dict into the post
|
||||
# record (they're already JSON scalars/objects). Everything lands in
|
||||
# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and
|
||||
@@ -150,9 +171,11 @@ class PixivDownloader(BaseNativeDownloader):
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
# The client's filename already carries the {id}_{title50}_{NN} shape;
|
||||
# only path-restrict sanitization happens here.
|
||||
media_path = flat_dir / sanitize_segment(media.filename)
|
||||
# The client's filename already carries the {id}_{title50}_{NN} shape
|
||||
# (raw title, gallery-dl-template order); clean it to the byte-exact
|
||||
# name gallery-dl wrote on disk so tier-2 disk-skip matches (else a
|
||||
# re-download of the whole work). See gdl_clean_filename.
|
||||
media_path = flat_dir / gdl_clean_filename(media.filename)
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
|
||||
Reference in New Issue
Block a user