feat(ingest): localize inline post-body images to local copies (Phase 2)
Render a post body faithfully by serving our stored copies of inline images instead of hotlinking the public CDN. The join key is the CDN filehash (32-hex MD5) shared between a body <img src> and the media URL we downloaded (the same identity extract_media dedups by): - utils.paths.filehash_from_url — one source of truth for the extractor; patreon_client._filehash now delegates so capture- and render-time hashing cannot drift. - ImageRecord gains source_url (provenance) + source_filehash (indexed match key); migration 0051. - the per-media sidecar carries the file's source_url; the importer persists it (NULL-only) on the ImageRecord via _apply_sidecar. - post_feed_service.get_post remaps body <img src> -> /images/<path> for every inline image whose filehash maps to a stored image of THIS artist; unmatched / pre-Phase-2 images keep hotlinking. Pre-existing on-disk images have no filehash yet, so they fall back to hotlinking until re-downloaded; localization is forward-looking. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import safe_ext
|
||||
from ..utils.paths import filehash_from_url, safe_ext
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -90,12 +90,6 @@ _FIELDS_POST = (
|
||||
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
|
||||
_FIELDS_CAMPAIGN = "name,url"
|
||||
|
||||
# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
|
||||
# Patreon's stable per-file identity and is what we dedup + ledger against.
|
||||
# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
|
||||
# in the URL (path or query); real Patreon CDN URLs carry exactly one.
|
||||
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
|
||||
|
||||
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
|
||||
# Pull every src; downstream dedup collapses any that duplicate a gallery item
|
||||
# by filehash. Tolerant of attribute ordering and single/double quotes.
|
||||
@@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
||||
|
||||
|
||||
def _filehash(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
match = _FILEHASH_RE.search(url)
|
||||
return match.group(1).lower() if match else None
|
||||
# Delegate to the shared extractor (utils.paths) so capture-time persistence
|
||||
# and render-time inline-image matching use the EXACT same identity.
|
||||
return filehash_from_url(url)
|
||||
|
||||
|
||||
def _basename_from_url(url: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user