feat(ingest): localize inline post-body images to local copies (Phase 2)
Render a post body faithfully by serving our stored copies of inline images instead of hotlinking the public CDN. The join key is the CDN filehash (32-hex MD5) shared between a body <img src> and the media URL we downloaded (the same identity extract_media dedups by): - utils.paths.filehash_from_url — one source of truth for the extractor; patreon_client._filehash now delegates so capture- and render-time hashing cannot drift. - ImageRecord gains source_url (provenance) + source_filehash (indexed match key); migration 0051. - the per-media sidecar carries the file's source_url; the importer persists it (NULL-only) on the ImageRecord via _apply_sidecar. - post_feed_service.get_post remaps body <img src> -> /images/<path> for every inline image whose filehash maps to a stored image of THIS artist; unmatched / pre-Phase-2 images keep hotlinking. Pre-existing on-disk images have no filehash yet, so they fall back to hotlinking until re-downloaded; localization is forward-looking. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,38 @@
|
|||||||
|
"""image_record: source_url + source_filehash (inline-image localization)
|
||||||
|
|
||||||
|
#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline
|
||||||
|
images instead of hotlinking the public CDN. The join key between a body
|
||||||
|
`<img src=CDN>` and the local file is the CDN's 32-hex filehash (the same
|
||||||
|
identity extract_media dedups by). Persist it (indexed) plus the full source
|
||||||
|
URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing
|
||||||
|
rows — those fall back to hotlinking until re-downloaded.
|
||||||
|
|
||||||
|
Revision ID: 0051
|
||||||
|
Revises: 0050
|
||||||
|
Create Date: 2026-06-14
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0051"
|
||||||
|
down_revision: Union[str, None] = "0050"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True))
|
||||||
|
op.add_column(
|
||||||
|
"image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True)
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_image_record_source_filehash", "image_record", ["source_filehash"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_image_record_source_filehash", table_name="image_record")
|
||||||
|
op.drop_column("image_record", "source_filehash")
|
||||||
|
op.drop_column("image_record", "source_url")
|
||||||
@@ -49,6 +49,18 @@ class ImageRecord(Base):
|
|||||||
# Thumbnail (populated by FC-2)
|
# Thumbnail (populated by FC-2)
|
||||||
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
|
# Source provenance for downloaded media (#830 Phase 2). `source_url` is the
|
||||||
|
# CDN/origin URL the file was fetched from (debugging + future re-fetch).
|
||||||
|
# `source_filehash` is the URL's 32-hex CDN identity segment
|
||||||
|
# (utils.paths.filehash_from_url) — the JOIN KEY that maps a post body's
|
||||||
|
# inline `<img src=CDN>` back to this local copy so the rendered body serves
|
||||||
|
# our stored image instead of hotlinking the public source. Indexed for the
|
||||||
|
# render-time lookup. NULL for filesystem-imported / pre-Phase-2 rows.
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
source_filehash: Mapped[str | None] = mapped_column(
|
||||||
|
String(32), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
# Origin / provenance pointers
|
# Origin / provenance pointers
|
||||||
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
|
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
|
||||||
primary_post_id: Mapped[int | None] = mapped_column(
|
primary_post_id: Mapped[int | None] = mapped_column(
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ from ..utils import safe_probe
|
|||||||
from ..utils.paths import (
|
from ..utils.paths import (
|
||||||
derive_subdir,
|
derive_subdir,
|
||||||
derive_top_level_artist,
|
derive_top_level_artist,
|
||||||
|
filehash_from_url,
|
||||||
hash_suffixed_name,
|
hash_suffixed_name,
|
||||||
safe_ext,
|
safe_ext,
|
||||||
)
|
)
|
||||||
@@ -1046,6 +1047,14 @@ class Importer:
|
|||||||
if record.artist_id is None:
|
if record.artist_id is None:
|
||||||
record.artist_id = artist.id
|
record.artist_id = artist.id
|
||||||
|
|
||||||
|
# #830 Phase 2: persist the file's CDN source identity (NULL-only, so a
|
||||||
|
# re-import / enrich-on-duplicate never clobbers it) — the filehash is
|
||||||
|
# the join key that lets post_feed_service remap the body's inline
|
||||||
|
# `<img src=CDN>` to this local copy at render time.
|
||||||
|
if sd.source_url and record.source_filehash is None:
|
||||||
|
record.source_url = sd.source_url
|
||||||
|
record.source_filehash = filehash_from_url(sd.source_url)
|
||||||
|
|
||||||
if explicit_source is not None:
|
if explicit_source is not None:
|
||||||
src = explicit_source
|
src = explicit_source
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ from urllib.parse import parse_qs, urlsplit
|
|||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from ..utils.paths import safe_ext
|
from ..utils.paths import filehash_from_url, safe_ext
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -90,12 +90,6 @@ _FIELDS_POST = (
|
|||||||
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
|
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
|
||||||
_FIELDS_CAMPAIGN = "name,url"
|
_FIELDS_CAMPAIGN = "name,url"
|
||||||
|
|
||||||
# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
|
|
||||||
# Patreon's stable per-file identity and is what we dedup + ledger against.
|
|
||||||
# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
|
|
||||||
# in the URL (path or query); real Patreon CDN URLs carry exactly one.
|
|
||||||
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
|
|
||||||
|
|
||||||
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
|
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
|
||||||
# Pull every src; downstream dedup collapses any that duplicate a gallery item
|
# Pull every src; downstream dedup collapses any that duplicate a gallery item
|
||||||
# by filehash. Tolerant of attribute ordering and single/double quotes.
|
# by filehash. Tolerant of attribute ordering and single/double quotes.
|
||||||
@@ -188,10 +182,9 @@ def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
|||||||
|
|
||||||
|
|
||||||
def _filehash(url: str) -> str | None:
|
def _filehash(url: str) -> str | None:
|
||||||
if not url:
|
# Delegate to the shared extractor (utils.paths) so capture-time persistence
|
||||||
return None
|
# and render-time inline-image matching use the EXACT same identity.
|
||||||
match = _FILEHASH_RE.search(url)
|
return filehash_from_url(url)
|
||||||
return match.group(1).lower() if match else None
|
|
||||||
|
|
||||||
|
|
||||||
def _basename_from_url(url: str) -> str:
|
def _basename_from_url(url: str) -> str:
|
||||||
|
|||||||
@@ -294,7 +294,7 @@ class PatreonDownloader:
|
|||||||
path=quarantine_dest, error=reason,
|
path=quarantine_dest, error=reason,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._write_sidecar(post, out_path)
|
self._write_sidecar(post, out_path, source_url=media.url)
|
||||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||||
|
|
||||||
# -- download seams ----------------------------------------------------
|
# -- download seams ----------------------------------------------------
|
||||||
@@ -490,7 +490,9 @@ class PatreonDownloader:
|
|||||||
|
|
||||||
# -- sidecar -----------------------------------------------------------
|
# -- sidecar -----------------------------------------------------------
|
||||||
|
|
||||||
def _write_sidecar(self, post: dict, media_path: Path) -> Path:
|
def _write_sidecar(
|
||||||
|
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||||
|
) -> Path:
|
||||||
"""Write the importer-consumed sidecar next to `media_path`.
|
"""Write the importer-consumed sidecar next to `media_path`.
|
||||||
|
|
||||||
Patreon uses base-default sidecar keys (parse_sidecar maps
|
Patreon uses base-default sidecar keys (parse_sidecar maps
|
||||||
@@ -498,13 +500,22 @@ class PatreonDownloader:
|
|||||||
content->description, published_at->post_date, url->post_url). Patreon
|
content->description, published_at->post_date, url->post_url). Patreon
|
||||||
registers no derive_post_url, so `url` is trusted as the permalink — we
|
registers no derive_post_url, so `url` is trusted as the permalink — we
|
||||||
pass the post's attributes.url.
|
pass the post's attributes.url.
|
||||||
"""
|
|
||||||
return self._write_sidecar_data(post, media_path.with_suffix(".json"))
|
|
||||||
|
|
||||||
def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
|
`source_url` is THIS file's CDN URL (#830 Phase 2). The importer persists
|
||||||
|
its filehash on the ImageRecord so the post body's inline `<img src>` can
|
||||||
|
be remapped to the local copy at render time.
|
||||||
|
"""
|
||||||
|
return self._write_sidecar_data(
|
||||||
|
post, media_path.with_suffix(".json"), source_url=source_url
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_sidecar_data(
|
||||||
|
self, post: dict, sidecar_path: Path, *, source_url: str | None = None
|
||||||
|
) -> Path:
|
||||||
"""Serialize the post's metadata to `sidecar_path`. Shared by the
|
"""Serialize the post's metadata to `sidecar_path`. Shared by the
|
||||||
per-media sidecar (next to each file) and the post-only sidecar
|
per-media sidecar (next to each file) and the post-only sidecar
|
||||||
(`write_post_record`, for media-less posts)."""
|
(`write_post_record`, for media-less posts). `source_url` is set only for
|
||||||
|
the per-media sidecar — a media-less post has no source file."""
|
||||||
attrs = post.get("attributes") or {}
|
attrs = post.get("attributes") or {}
|
||||||
title = attrs.get("title")
|
title = attrs.get("title")
|
||||||
content = attrs.get("content")
|
content = attrs.get("content")
|
||||||
@@ -530,6 +541,8 @@ class PatreonDownloader:
|
|||||||
"published_at": published if isinstance(published, str) else None,
|
"published_at": published if isinstance(published, str) else None,
|
||||||
"url": url if isinstance(url, str) else None,
|
"url": url if isinstance(url, str) else None,
|
||||||
}
|
}
|
||||||
|
if source_url:
|
||||||
|
data["source_url"] = source_url
|
||||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||||
return sidecar_path
|
return sidecar_path
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ attachments from PostAttachment) so the API layer can jsonify directly.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
from sqlalchemy import and_, func, or_, select
|
from sqlalchemy import and_, func, or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
@@ -23,9 +25,14 @@ from ..models import (
|
|||||||
PostAttachment,
|
PostAttachment,
|
||||||
Source,
|
Source,
|
||||||
)
|
)
|
||||||
from ..utils.html_sanitize import sanitize_post_html
|
from ..utils.html_sanitize import (
|
||||||
|
extract_img_srcs,
|
||||||
|
rewrite_img_srcs,
|
||||||
|
sanitize_post_html,
|
||||||
|
)
|
||||||
|
from ..utils.paths import filehash_from_url
|
||||||
from ..utils.text import html_to_plain, truncate_at_word
|
from ..utils.text import html_to_plain, truncate_at_word
|
||||||
from .gallery_service import thumbnail_url
|
from .gallery_service import image_url, thumbnail_url
|
||||||
from .pagination import decode_cursor, encode_cursor
|
from .pagination import decode_cursor, encode_cursor
|
||||||
|
|
||||||
DESCRIPTION_LIMIT = 280
|
DESCRIPTION_LIMIT = 280
|
||||||
@@ -195,11 +202,52 @@ class PostFeedService:
|
|||||||
item["description_full"] = html_to_plain(post.description)
|
item["description_full"] = html_to_plain(post.description)
|
||||||
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
||||||
# detail-only (the feed list stays lightweight plain text). None when the
|
# detail-only (the feed list stays lightweight plain text). None when the
|
||||||
# post has no body.
|
# post has no body. Inline `<img>` sources are remapped to locally-served
|
||||||
item["description_html"] = sanitize_post_html(post.description)
|
# copies (#830 Phase 2) so the body never hotlinks the public CDN.
|
||||||
|
item["description_html"] = await self._localize_inline_images(
|
||||||
|
sanitize_post_html(post.description), post.artist_id,
|
||||||
|
)
|
||||||
item["external_links"] = await self._external_links_for(post.id)
|
item["external_links"] = await self._external_links_for(post.id)
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
async def _localize_inline_images(
|
||||||
|
self, html: str | None, artist_id: int | None,
|
||||||
|
) -> str | None:
|
||||||
|
"""Rewrite a post body's inline `<img src=CDN>` to locally-served copies.
|
||||||
|
|
||||||
|
The join key is the CDN filehash the downloader persisted on each
|
||||||
|
ImageRecord (source_filehash): for every body image whose filehash maps
|
||||||
|
to a stored image of THIS artist, swap the src to /images/<path>. Images
|
||||||
|
we never captured (or pre-Phase-2 rows with no filehash) are left as-is —
|
||||||
|
they keep hotlinking, which is the prior behavior. Scoped to the post's
|
||||||
|
artist so one creator's body never resolves to another's file."""
|
||||||
|
if not html or artist_id is None:
|
||||||
|
return html
|
||||||
|
srcs = extract_img_srcs(html)
|
||||||
|
if not srcs:
|
||||||
|
return html
|
||||||
|
# filehash -> the raw (as-in-HTML) src strings carrying it. A body can
|
||||||
|
# repeat the same image; keep every raw form so each is substituted.
|
||||||
|
by_hash: dict[str, list[str]] = {}
|
||||||
|
for raw in srcs:
|
||||||
|
fh = filehash_from_url(unescape(raw))
|
||||||
|
if fh:
|
||||||
|
by_hash.setdefault(fh, []).append(raw)
|
||||||
|
if not by_hash:
|
||||||
|
return html
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(ImageRecord.source_filehash, ImageRecord.path)
|
||||||
|
.where(
|
||||||
|
ImageRecord.artist_id == artist_id,
|
||||||
|
ImageRecord.source_filehash.in_(list(by_hash)),
|
||||||
|
)
|
||||||
|
)).all()
|
||||||
|
replace: dict[str, str] = {}
|
||||||
|
for fh, path in rows:
|
||||||
|
for raw in by_hash.get(fh, ()):
|
||||||
|
replace[raw] = image_url(path)
|
||||||
|
return rewrite_img_srcs(html, replace)
|
||||||
|
|
||||||
async def _external_links_for(self, post_id: int) -> list[dict]:
|
async def _external_links_for(self, post_id: int) -> list[dict]:
|
||||||
"""Off-platform file-host links recorded for a post (detail-only). Each
|
"""Off-platform file-host links recorded for a post (detail-only). Each
|
||||||
carries its host, full url, label, and download status so the post view
|
carries its host, full url, label, and download status so the post view
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used;
|
|||||||
bleach is deprecated upstream and intentionally not used.
|
bleach is deprecated upstream and intentionally not used.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
import nh3
|
import nh3
|
||||||
|
|
||||||
# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
|
# A faithful-but-safe set: enough to reproduce the SEMANTIC look of a scraped
|
||||||
@@ -50,3 +52,39 @@ def sanitize_post_html(raw: str | None) -> str | None:
|
|||||||
link_rel="noopener noreferrer",
|
link_rel="noopener noreferrer",
|
||||||
clean_content_tags={"script", "style"},
|
clean_content_tags={"script", "style"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# nh3 emits attributes double-quoted, so a single `src="..."` capture is enough
|
||||||
|
# to read/rewrite inline-image sources on already-sanitized markup (#830 Phase 2).
|
||||||
|
_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc=")([^"]*)(")', re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_img_srcs(html: str | None) -> list[str]:
|
||||||
|
"""The raw `src` values of every <img> in `html`, in document order, deduped
|
||||||
|
(first kept). Returned EXACTLY as they appear (still entity-escaped) so a
|
||||||
|
caller can match them back for verbatim substitution."""
|
||||||
|
if not html:
|
||||||
|
return []
|
||||||
|
out: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for _pre, src, _post in _IMG_SRC_RE.findall(html):
|
||||||
|
if src and src not in seen:
|
||||||
|
seen.add(src)
|
||||||
|
out.append(src)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_img_srcs(html: str | None, replace: dict[str, str]) -> str | None:
|
||||||
|
"""Swap each <img src> whose raw (as-in-HTML) value is a key in `replace`
|
||||||
|
with the mapped value; others are left untouched. Used to point a post body's
|
||||||
|
inline images at locally-served copies (#830 Phase 2). Operates only on src
|
||||||
|
values of already-sanitized markup, so the result stays within the allowlist.
|
||||||
|
Returns `html` unchanged when it or `replace` is empty."""
|
||||||
|
if not html or not replace:
|
||||||
|
return html
|
||||||
|
|
||||||
|
def _sub(m: re.Match) -> str:
|
||||||
|
pre, src, post = m.group(1), m.group(2), m.group(3)
|
||||||
|
return f"{pre}{replace.get(src, src)}{post}"
|
||||||
|
|
||||||
|
return _IMG_SRC_RE.sub(_sub, html)
|
||||||
|
|||||||
@@ -1,9 +1,28 @@
|
|||||||
"""Filesystem path helpers — destination derivation, hash-suffixed names."""
|
"""Filesystem path helpers — destination derivation, hash-suffixed names."""
|
||||||
|
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
_MAX_EXT_LEN = 16
|
_MAX_EXT_LEN = 16
|
||||||
|
|
||||||
|
# A Patreon/gallery CDN URL embeds a 32-char hex (MD5) path segment that is the
|
||||||
|
# file's stable per-file identity — the same role gallery-dl's `_filehash`
|
||||||
|
# plays. It is the join key between a post body `<img src=CDN>` and the local
|
||||||
|
# copy we downloaded (extract_media dedups content vs gallery images by it), so
|
||||||
|
# this ONE extractor must be used for both capture-time persistence and
|
||||||
|
# render-time matching — they cannot be allowed to drift. Match the FIRST 32-hex
|
||||||
|
# run anywhere in the URL (path or query); real CDN URLs carry exactly one.
|
||||||
|
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
|
||||||
|
|
||||||
|
|
||||||
|
def filehash_from_url(url: str | None) -> str | None:
|
||||||
|
"""The 32-char hex (MD5) CDN identity segment of `url`, lowercased, or None
|
||||||
|
when the URL is empty / carries no such segment."""
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
match = _FILEHASH_RE.search(url)
|
||||||
|
return match.group(1).lower() if match else None
|
||||||
|
|
||||||
|
|
||||||
def safe_ext(name: str | Path) -> str:
|
def safe_ext(name: str | Path) -> str:
|
||||||
"""Conservatively extract a short, alphanumeric file extension.
|
"""Conservatively extract a short, alphanumeric file extension.
|
||||||
|
|||||||
@@ -27,6 +27,11 @@ class SidecarData:
|
|||||||
description: str | None
|
description: str | None
|
||||||
attachment_count: int | None
|
attachment_count: int | None
|
||||||
post_date: datetime | None
|
post_date: datetime | None
|
||||||
|
# Per-FILE CDN/origin URL the media was downloaded from (#830 Phase 2). Only
|
||||||
|
# the per-media sidecar carries it; post-only sidecars leave it None. The
|
||||||
|
# importer persists it on the ImageRecord so the body's inline `<img src>`
|
||||||
|
# can be matched (by filehash) to the local copy at render time.
|
||||||
|
source_url: str | None
|
||||||
raw: dict
|
raw: dict
|
||||||
|
|
||||||
|
|
||||||
@@ -172,6 +177,11 @@ def parse_sidecar(data: dict) -> SidecarData:
|
|||||||
else:
|
else:
|
||||||
post_url = _first_str(data, ("url", "post_url"))
|
post_url = _first_str(data, ("url", "post_url"))
|
||||||
|
|
||||||
|
# Per-file source URL (#830 Phase 2): the native downloader writes it into
|
||||||
|
# each media sidecar. `url` is the post permalink for Patreon (handled
|
||||||
|
# above), so source_url has its own dedicated key and never collides.
|
||||||
|
source_url = _first_str(data, ("source_url",))
|
||||||
|
|
||||||
return SidecarData(
|
return SidecarData(
|
||||||
platform=platform,
|
platform=platform,
|
||||||
external_post_id=external_post_id,
|
external_post_id=external_post_id,
|
||||||
@@ -180,5 +190,6 @@ def parse_sidecar(data: dict) -> SidecarData:
|
|||||||
description=description,
|
description=description,
|
||||||
attachment_count=attachment_count,
|
attachment_count=attachment_count,
|
||||||
post_date=post_date,
|
post_date=post_date,
|
||||||
|
source_url=source_url,
|
||||||
raw=data,
|
raw=data,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
from backend.app.utils.html_sanitize import sanitize_post_html
|
from backend.app.utils.html_sanitize import (
|
||||||
|
extract_img_srcs,
|
||||||
|
rewrite_img_srcs,
|
||||||
|
sanitize_post_html,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_none_and_blank_return_none():
|
def test_none_and_blank_return_none():
|
||||||
@@ -52,3 +56,40 @@ def test_headings_lists_and_images_survive():
|
|||||||
def test_image_javascript_src_dropped():
|
def test_image_javascript_src_dropped():
|
||||||
out = sanitize_post_html('<img src="javascript:alert(1)" alt="x">')
|
out = sanitize_post_html('<img src="javascript:alert(1)" alt="x">')
|
||||||
assert "javascript:" not in out
|
assert "javascript:" not in out
|
||||||
|
|
||||||
|
|
||||||
|
# --- inline-image localization helpers (#830 Phase 2) ----------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_img_srcs_in_order_deduped():
|
||||||
|
html = (
|
||||||
|
'<p>a</p><img src="https://cdn.test/1.jpg">'
|
||||||
|
'<img alt="x" src="https://cdn.test/2.jpg" width="10">'
|
||||||
|
'<img src="https://cdn.test/1.jpg">'
|
||||||
|
)
|
||||||
|
assert extract_img_srcs(html) == [
|
||||||
|
"https://cdn.test/1.jpg",
|
||||||
|
"https://cdn.test/2.jpg",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_img_srcs_empty():
|
||||||
|
assert extract_img_srcs(None) == []
|
||||||
|
assert extract_img_srcs("<p>no images</p>") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_rewrite_img_srcs_swaps_only_mapped():
|
||||||
|
html = (
|
||||||
|
'<img src="https://cdn.test/1.jpg" alt="a">'
|
||||||
|
'<img src="https://cdn.test/2.jpg">'
|
||||||
|
)
|
||||||
|
out = rewrite_img_srcs(html, {"https://cdn.test/1.jpg": "/images/local/1.jpg"})
|
||||||
|
assert 'src="/images/local/1.jpg"' in out
|
||||||
|
assert 'alt="a"' in out # other attributes preserved
|
||||||
|
assert 'src="https://cdn.test/2.jpg"' in out # unmapped left alone
|
||||||
|
|
||||||
|
|
||||||
|
def test_rewrite_img_srcs_noop_on_empty():
|
||||||
|
html = '<img src="https://cdn.test/1.jpg">'
|
||||||
|
assert rewrite_img_srcs(html, {}) == html
|
||||||
|
assert rewrite_img_srcs(None, {"a": "b"}) is None
|
||||||
|
|||||||
@@ -3,12 +3,31 @@ from pathlib import Path
|
|||||||
from backend.app.utils.paths import (
|
from backend.app.utils.paths import (
|
||||||
derive_subdir,
|
derive_subdir,
|
||||||
derive_top_level_artist,
|
derive_top_level_artist,
|
||||||
|
filehash_from_url,
|
||||||
hash_suffixed_name,
|
hash_suffixed_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
IMPORT = Path("/import")
|
IMPORT = Path("/import")
|
||||||
|
|
||||||
|
|
||||||
|
def test_filehash_from_url_extracts_lowercased_md5():
|
||||||
|
url = "https://c10.patreonusercontent.com/4/patreon-media/p/post/1/AbC123DeF456789012345678901234EF/1/img.png?token=x"
|
||||||
|
assert filehash_from_url(url) == "abc123def456789012345678901234ef"
|
||||||
|
|
||||||
|
|
||||||
|
def test_filehash_from_url_none_and_no_hash():
|
||||||
|
assert filehash_from_url(None) is None
|
||||||
|
assert filehash_from_url("") is None
|
||||||
|
assert filehash_from_url("https://example.test/a.png") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_filehash_from_url_query_unescaping_irrelevant():
|
||||||
|
# The hash sits in the path before the query, so entity-escaped ampersands
|
||||||
|
# in the query never affect the match (render-time matches escaped srcs).
|
||||||
|
base = "https://cdn.test/p/0123456789abcdef0123456789ABCDEF/x.jpg"
|
||||||
|
assert filehash_from_url(base + "?a=1&b=2") == "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
def test_derive_subdir_nested():
|
def test_derive_subdir_nested():
|
||||||
assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub"
|
assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub"
|
||||||
|
|
||||||
|
|||||||
@@ -175,6 +175,9 @@ def test_sidecar_written_and_findable(tmp_path):
|
|||||||
assert data["id"] == "1001"
|
assert data["id"] == "1001"
|
||||||
assert data["title"] == "My Post Title"
|
assert data["title"] == "My Post Title"
|
||||||
assert data["url"] == "https://www.patreon.com/posts/1001"
|
assert data["url"] == "https://www.patreon.com/posts/1001"
|
||||||
|
# #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the
|
||||||
|
# importer can persist its filehash for inline-image localization.
|
||||||
|
assert data["source_url"] == "https://cdn.patreon.com/media1.png"
|
||||||
|
|
||||||
|
|
||||||
def test_skip_seen(tmp_path):
|
def test_skip_seen(tmp_path):
|
||||||
|
|||||||
@@ -438,6 +438,62 @@ async def test_get_post_returns_sanitized_html_body(db):
|
|||||||
assert "<script>" not in item["description_html"]
|
assert "<script>" not in item["description_html"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_post_localizes_inline_images(db):
|
||||||
|
"""#830 Phase 2: a body <img src=CDN> whose filehash matches a stored image
|
||||||
|
of this artist is rewritten to the local /images path; an unmatched src is
|
||||||
|
left hotlinking."""
|
||||||
|
artist = await _seed_artist(db, "alice-inline")
|
||||||
|
src = await _seed_source(db, artist.id, "patreon", "https://p/alice-inline")
|
||||||
|
fh = "0123456789abcdef0123456789abcdef"
|
||||||
|
matched = f"https://cdn.test/p/{fh}/img.png"
|
||||||
|
unmatched = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/other.png"
|
||||||
|
p = await _seed_post(
|
||||||
|
db, src.id, external_id="INLINE", post_date=datetime.now(UTC),
|
||||||
|
description=f'<p>x</p><img src="{matched}"><img src="{unmatched}">',
|
||||||
|
)
|
||||||
|
db.add(ImageRecord(
|
||||||
|
path="/images/alice-inline/patreon/post/01_img.png",
|
||||||
|
sha256=f"{p.id:064d}",
|
||||||
|
size_bytes=10, mime="image/png", width=10, height=10,
|
||||||
|
origin="downloaded", primary_post_id=p.id, artist_id=artist.id,
|
||||||
|
source_url=matched, source_filehash=fh,
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
item = await PostFeedService(db).get_post(p.id)
|
||||||
|
html = item["description_html"]
|
||||||
|
assert 'src="/images/alice-inline/patreon/post/01_img.png"' in html
|
||||||
|
assert matched not in html # CDN src swapped out
|
||||||
|
assert unmatched in html # no local copy → still hotlinked
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_post_inline_image_scoped_to_artist(db):
|
||||||
|
"""A matching filehash owned by a DIFFERENT artist must not leak into this
|
||||||
|
post's body — localization is artist-scoped."""
|
||||||
|
a1 = await _seed_artist(db, "owner-art")
|
||||||
|
a2 = await _seed_artist(db, "other-art")
|
||||||
|
src = await _seed_source(db, a1.id, "patreon", "https://p/owner-art")
|
||||||
|
fh = "abcabcabcabcabcabcabcabcabcabc12"
|
||||||
|
cdn = f"https://cdn.test/p/{fh}/x.png"
|
||||||
|
p = await _seed_post(
|
||||||
|
db, src.id, external_id="SCOPED", post_date=datetime.now(UTC),
|
||||||
|
description=f'<img src="{cdn}">',
|
||||||
|
)
|
||||||
|
# The only image with this filehash belongs to a DIFFERENT artist.
|
||||||
|
db.add(ImageRecord(
|
||||||
|
path="/images/other/x.png", sha256=f"{p.id:064d}",
|
||||||
|
size_bytes=10, mime="image/png", width=10, height=10,
|
||||||
|
origin="downloaded", artist_id=a2.id,
|
||||||
|
source_url=cdn, source_filehash=fh,
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
item = await PostFeedService(db).get_post(p.id)
|
||||||
|
assert cdn in item["description_html"] # not localized (wrong artist)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_get_post_returns_external_links(db):
|
async def test_get_post_returns_external_links(db):
|
||||||
artist = await _seed_artist(db, "alice-extlinks")
|
artist = await _seed_artist(db, "alice-extlinks")
|
||||||
|
|||||||
@@ -101,6 +101,25 @@ def test_sidecar_creates_provenance(importer, import_layout):
|
|||||||
assert rec.effective_date == post.post_date
|
assert rec.effective_date == post.post_date
|
||||||
|
|
||||||
|
|
||||||
|
def test_sidecar_source_url_persists_filehash(importer, import_layout):
|
||||||
|
"""#830 Phase 2: a media sidecar's source_url lands on the ImageRecord as
|
||||||
|
source_url + the lowercased CDN filehash (the inline-image join key)."""
|
||||||
|
import_root, _ = import_layout
|
||||||
|
m = import_root / "Alice" / "b.jpg"
|
||||||
|
_split(m, "v")
|
||||||
|
_sidecar(m, {
|
||||||
|
"category": "patreon", "id": 901, "title": "Body post",
|
||||||
|
"source_url": "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg",
|
||||||
|
})
|
||||||
|
r = importer.import_one(m)
|
||||||
|
assert r.status == "imported"
|
||||||
|
rec = importer.session.get(ImageRecord, r.image_id)
|
||||||
|
assert rec.source_url == (
|
||||||
|
"https://cdn.test/p/0123456789ABCDEF0123456789abcdef/b.jpg"
|
||||||
|
)
|
||||||
|
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
def test_reimport_same_post_idempotent(importer, import_layout):
|
def test_reimport_same_post_idempotent(importer, import_layout):
|
||||||
import_root, _ = import_layout
|
import_root, _ = import_layout
|
||||||
# Threshold 0: only an exact phash match collapses. Orthogonal splits
|
# Threshold 0: only an exact phash match collapses. Orthogonal splits
|
||||||
|
|||||||
@@ -119,6 +119,15 @@ def test_parse_message_used_as_description_fallback():
|
|||||||
assert sd.description == "hello channel"
|
assert sd.description == "hello channel"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_source_url_present_and_absent():
|
||||||
|
"""The per-media sidecar carries a `source_url` (#830 Phase 2); a post-only
|
||||||
|
sidecar (no media file) omits it → None."""
|
||||||
|
sd = parse_sidecar({"category": "patreon", "id": "1",
|
||||||
|
"source_url": "https://cdn.test/abc.png"})
|
||||||
|
assert sd.source_url == "https://cdn.test/abc.png"
|
||||||
|
assert parse_sidecar({"category": "patreon", "id": "1"}).source_url is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_date_epoch_and_unparseable_and_naive():
|
def test_parse_date_epoch_and_unparseable_and_naive():
|
||||||
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
|
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
|
||||||
assert parse_sidecar({"date": "not-a-date"}).post_date is None
|
assert parse_sidecar({"date": "not-a-date"}).post_date is None
|
||||||
|
|||||||
Reference in New Issue
Block a user