feat(ingest): localize inline post-body images to local copies (Phase 2)
Render a post body faithfully by serving our stored copies of inline images instead of hotlinking the public CDN. The join key is the CDN filehash (32-hex MD5) shared between a body <img src> and the media URL we downloaded (the same identity extract_media dedups by): - utils.paths.filehash_from_url — one source of truth for the extractor; patreon_client._filehash now delegates so capture- and render-time hashing cannot drift. - ImageRecord gains source_url (provenance) + source_filehash (indexed match key); migration 0051. - the per-media sidecar carries the file's source_url; the importer persists it (NULL-only) on the ImageRecord via _apply_sidecar. - post_feed_service.get_post remaps body <img src> -> /images/<path> for every inline image whose filehash maps to a stored image of THIS artist; unmatched / pre-Phase-2 images keep hotlinking. Pre-existing on-disk images have no filehash yet, so they fall back to hotlinking until re-downloaded; localization is forward-looking. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used;
|
||||
bleach is deprecated upstream and intentionally not used.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import nh3
|
||||
|
||||
# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
|
||||
@@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None:
|
||||
link_rel="noopener noreferrer",
|
||||
clean_content_tags={"script", "style"},
|
||||
)
|
||||
|
||||
|
||||
# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough
|
||||
# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2).
|
||||
_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc=")([^"]*)(")', re.IGNORECASE)
|
||||
|
||||
|
||||
def extract_img_srcs(html: str | None) -> list[str]:
|
||||
"""The raw `src` values of every <img> in `html`, in document order, deduped
|
||||
(first kept). Returned EXACTLY as they appear (still entity-escaped) so a
|
||||
caller can match them back for verbatim substitution."""
|
||||
if not html:
|
||||
return []
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for _pre, src, _post in _IMG_SRC_RE.findall(html):
|
||||
if src and src not in seen:
|
||||
seen.add(src)
|
||||
out.append(src)
|
||||
return out
|
||||
|
||||
|
||||
def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None:
|
||||
"""Swap each <img src> whose raw (as-in-HTML) value is a key in `replace`
|
||||
with the mapped value; others are left untouched. Used to point a post body's
|
||||
inline images at locally-served copies (#830 Phase 2). Operates only on src
|
||||
values of already-sanitized markup, so the result stays within the allowlist.
|
||||
Returns `html` unchanged when it or `replace` is empty."""
|
||||
if not html or not replace:
|
||||
return html
|
||||
|
||||
def _sub(m: re.Match) -> str:
|
||||
pre, src, post = m.group(1), m.group(2), m.group(3)
|
||||
return f"{pre}{replace.get(src, src)}{post}"
|
||||
|
||||
return _IMG_SRC_RE.sub(_sub, html)
|
||||
|
||||
@@ -1,9 +1,28 @@
|
||||
"""Filesystem path helpers — destination derivation, hash-suffixed names."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_MAX_EXT_LEN = 16
|
||||
|
||||
# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the
|
||||
# file's stable per-file identity — the same role gallery-dl's `_filehash`
|
||||
# plays. It is the join key between a post body `<img src=CDN>` and the local
|
||||
# copy we downloaded (extract_media dedups content vs gallery images by it), so
|
||||
# this ONE extractor must be used for both capture-time persistence and
|
||||
# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex
|
||||
# run anywhere in the URL (path or query); real CDN URLs carry exactly one.
|
||||
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
|
||||
|
||||
|
||||
def filehash_from_url(url: str | None) -> str | None:
|
||||
"""The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None
|
||||
when the URL is empty / carries no such segment."""
|
||||
if not url:
|
||||
return None
|
||||
match = _FILEHASH_RE.search(url)
|
||||
return match.group(1).lower() if match else None
|
||||
|
||||
|
||||
def safe_ext(name: str | Path) -> str:
|
||||
"""Conservatively extract a short, alphanumeric file extension.
|
||||
|
||||
@@ -27,6 +27,11 @@ class SidecarData:
|
||||
description: str | None
|
||||
attachment_count: int | None
|
||||
post_date: datetime | None
|
||||
# Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only
|
||||
# the per-media sidecar carries it; post-only sidecars leave it None. The
|
||||
# importer persists it on the ImageRecord so the body's inline `<img src>`
|
||||
# can be matched (by filehash) to the local copy at render time.
|
||||
source_url: str | None
|
||||
raw: dict
|
||||
|
||||
|
||||
@@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData:
|
||||
else:
|
||||
post_url = _first_str(data, ("url", "post_url"))
|
||||
|
||||
# Per-file source URL (#830 Phase 2): the native downloader writes it into
|
||||
# each media sidecar. `url` is the post permalink for Patreon (handled
|
||||
# above), so source_url has its own dedicated key and never collides.
|
||||
source_url = _first_str(data, ("source_url",))
|
||||
|
||||
return SidecarData(
|
||||
platform=platform,
|
||||
external_post_id=external_post_id,
|
||||
@@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData:
|
||||
description=description,
|
||||
attachment_count=attachment_count,
|
||||
post_date=post_date,
|
||||
source_url=source_url,
|
||||
raw=data,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user