feat(patreon): native media downloader — ingester build step 2b (plan #697)
PatreonDownloader.download_post: writes resolved MediaItems to gallery-dl's exact on-disk layout (<slug>/patreon/<date>_<id>_<title40>/<NN>_<file>) + a sidecar the importer's find_sidecar/parse_sidecar consume unchanged. Two-tier skip (injected seen predicate, then disk). Streamed GET (.part→rename) + file_validator quarantine; Mux/m3u8 video shells out to yt-dlp with Patreon Referer/Origin. Pure (no DB) — ledger + orchestration land in step 3. Unit tests stub the session + yt-dlp seams (no network/subprocess). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,384 @@
|
||||
"""Native Patreon media downloader (build step 2b of the native ingester).
|
||||
|
||||
Given a Patreon post and its already-resolved `MediaItem`s (from
|
||||
patreon_client.extract_media), download the media to the EXACT on-disk layout
|
||||
gallery-dl produces, write a sidecar JSON the existing importer consumes, and
|
||||
report per-media outcomes.
|
||||
|
||||
This module is PURE: no DB. The cross-run seen-ledger and the iter_posts
|
||||
orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate
|
||||
(`is_seen`), so this module needs no DB and is unit-testable without network.
|
||||
|
||||
On-disk layout (matches gallery-dl):
|
||||
<images_root>/<artist_slug>/patreon/<DIR>/<NN>_<filename>
|
||||
where <DIR> = "<YYYY-MM-DD>_<post_id>_<title40>" (date prefix omitted when the
|
||||
post's published_at is unparseable), <NN> is the 1-based index of the item in
|
||||
the post zero-padded to 2, and <filename> is MediaItem.filename. The sidecar
|
||||
is written next to the media as <NN>_<stem>.json (media_path.with_suffix).
|
||||
|
||||
Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path
|
||||
endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same
|
||||
Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py
|
||||
_get_default_config). yt-dlp may remux to a container of its own choosing, so we
|
||||
accept the actual output extension and record the real path.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from .file_validator import is_validatable, validate_file
|
||||
from .patreon_client import _load_session
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TITLE_MAX = 40
|
||||
_TIMEOUT_SECONDS = 120.0
|
||||
_CHUNK = 1 << 16
|
||||
|
||||
# Same Referer/Origin gallery-dl forwards to yt-dlp for Mux-hosted Patreon
|
||||
# video (gallery_dl.py _get_default_config -> downloader.ytdl.raw-options).
|
||||
# Mux's JWT playback policy checks Referer/Origin on every request, so yt-dlp
|
||||
# must send Patreon's, not its own default.
|
||||
_VIDEO_HEADERS = {
|
||||
"Referer": "https://www.patreon.com/",
|
||||
"Origin": "https://www.patreon.com",
|
||||
}
|
||||
|
||||
# Characters Windows/gallery-dl path-restrict forbids, plus path separators.
|
||||
_FORBIDDEN = set('<>:"/\\|?*')
|
||||
|
||||
|
||||
def _sanitize(name: str) -> str:
|
||||
"""Make `name` safe for a single filesystem path segment.
|
||||
|
||||
Replaces path separators, the Windows-forbidden set <>:"/\\|?* and control
|
||||
characters with `_`, then strips trailing dots/spaces (gallery-dl
|
||||
path-restrict behavior). Never returns empty (falls back to "_").
|
||||
"""
|
||||
out = []
|
||||
for ch in name:
|
||||
if ch in _FORBIDDEN or ord(ch) < 32:
|
||||
out.append("_")
|
||||
else:
|
||||
out.append(ch)
|
||||
cleaned = "".join(out).rstrip(". ")
|
||||
return cleaned or "_"
|
||||
|
||||
|
||||
def _is_video_url(url: str) -> bool:
|
||||
parts = urlsplit(url)
|
||||
if parts.hostname and parts.hostname.lower() == "stream.mux.com":
|
||||
return True
|
||||
return parts.path.lower().endswith(".m3u8")
|
||||
|
||||
|
||||
def _post_dir_name(post: dict) -> str:
|
||||
"""Build the post directory name matching gallery-dl's layout."""
|
||||
post_id = str(post.get("id") or "")
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
title = title if isinstance(title, str) else ""
|
||||
title40 = title[:_TITLE_MAX]
|
||||
|
||||
published = attrs.get("published_at")
|
||||
date_prefix = None
|
||||
if isinstance(published, str) and published:
|
||||
s = published.strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
dt = None
|
||||
if dt is not None:
|
||||
date_prefix = f"{dt:%Y-%m-%d}"
|
||||
|
||||
if date_prefix:
|
||||
raw = f"{date_prefix}_{post_id}_{title40}"
|
||||
else:
|
||||
raw = f"{post_id}_{title40}"
|
||||
return _sanitize(raw)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaOutcome:
|
||||
"""Per-media result of a download_post pass.
|
||||
|
||||
status is one of: "downloaded", "skipped_seen", "skipped_disk", "error".
|
||||
`path` is the final on-disk path for "downloaded" (the actual yt-dlp output
|
||||
for video), or the path that already existed for "skipped_disk"; None for
|
||||
"skipped_seen" and (usually) "error". `error` carries the failure reason for
|
||||
"error", else None.
|
||||
"""
|
||||
|
||||
media: object # MediaItem (avoid importing the name for a bare annotation)
|
||||
status: str
|
||||
path: Path | None
|
||||
error: str | None
|
||||
|
||||
|
||||
class PatreonDownloader:
|
||||
"""Download resolved Patreon media to gallery-dl's on-disk layout.
|
||||
|
||||
PURE: no DB. The HTTP session and the yt-dlp invocation are injectable seams
|
||||
so tests run without network or a real subprocess:
|
||||
- pass `session=` to stub `session.get`, or monkeypatch `_fetch_to_file`.
|
||||
- monkeypatch `_run_ytdlp` to avoid spawning yt-dlp.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._validate = validate
|
||||
# Build a cookie-loaded session the same way patreon_client does, so the
|
||||
# CDN GETs carry the creator's auth.
|
||||
self.session = session if session is not None else _load_session(cookies_path)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
|
||||
Builds the post directory, iterates media (1-based NN), applies the
|
||||
two-tier skip (injected is_seen, then disk), downloads (plain GET or
|
||||
yt-dlp for video), writes the sidecar for each freshly-downloaded item,
|
||||
validates, and returns outcomes. Resilient: one media's failure yields
|
||||
an "error" outcome for that item; the rest proceed.
|
||||
"""
|
||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
|
||||
for i, media in enumerate(media_items, start=1):
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(post, media, post_dir, artist_slug, i, is_seen)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Patreon media failed (post %s, item %d): %s",
|
||||
post.get("id"), i, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
post_dir: Path,
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[object], bool],
|
||||
) -> MediaOutcome:
|
||||
# tier-1: seen ledger (injected; no DB here).
|
||||
if is_seen(media):
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
final_name = _sanitize(f"{nn}_{media.filename}")
|
||||
media_path = post_dir / final_name
|
||||
|
||||
# tier-2: already on disk.
|
||||
if media_path.exists():
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
)
|
||||
# Video may land at a different extension; honor a pre-existing remux.
|
||||
if _is_video_url(media.url):
|
||||
existing = self._existing_video_output(media_path)
|
||||
if existing is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=existing, error=None
|
||||
)
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if _is_video_url(media.url):
|
||||
out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
|
||||
if out_path is None or not Path(out_path).exists():
|
||||
return MediaOutcome(
|
||||
media=media,
|
||||
status="error",
|
||||
path=None,
|
||||
error="yt-dlp produced no output",
|
||||
)
|
||||
out_path = Path(out_path)
|
||||
else:
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
invalid = self._validate_path(out_path, artist_slug)
|
||||
if invalid is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="error", path=None, error=invalid
|
||||
)
|
||||
|
||||
self._write_sidecar(post, out_path)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- download seams ----------------------------------------------------
|
||||
|
||||
def _fetch_get(self, url: str, dest: Path) -> Path:
|
||||
"""Stream `url` to a .part file then atomic-rename to `dest`.
|
||||
|
||||
Thin wrapper over `_fetch_to_file` so tests can stub either the whole
|
||||
GET path (`_fetch_to_file`) or just `session.get`.
|
||||
"""
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
try:
|
||||
self._fetch_to_file(url, part)
|
||||
except Exception:
|
||||
with contextlib.suppress(OSError):
|
||||
part.unlink()
|
||||
raise
|
||||
os.replace(part, dest)
|
||||
return dest
|
||||
|
||||
def _fetch_to_file(self, url: str, dest: Path) -> None:
|
||||
"""Stream a non-video URL to `dest` via the (stubbable) session."""
|
||||
resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS)
|
||||
resp.raise_for_status()
|
||||
with open(dest, "wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=_CHUNK):
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
|
||||
def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
|
||||
"""Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
|
||||
|
||||
Returns the actual output path (yt-dlp may remux to a different
|
||||
container, so we resolve the real file afterward). Overridden/
|
||||
monkeypatched in tests to avoid spawning a real subprocess.
|
||||
|
||||
The output template uses `dest` without its extension; yt-dlp appends
|
||||
the chosen container extension. We pass Referer/Origin (Mux JWT policy)
|
||||
and the cookies file.
|
||||
"""
|
||||
dest = Path(dest)
|
||||
out_template = str(dest.with_suffix("")) + ".%(ext)s"
|
||||
cmd = ["yt-dlp", "--no-progress", "-o", out_template]
|
||||
for key, value in headers.items():
|
||||
cmd += ["--add-header", f"{key}:{value}"]
|
||||
if self.cookies_path and os.path.isfile(self.cookies_path):
|
||||
cmd += ["--cookies", self.cookies_path]
|
||||
cmd.append(url)
|
||||
try:
|
||||
subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
log.warning("yt-dlp failed for %s: %s", url, exc)
|
||||
return None
|
||||
return self._existing_video_output(dest)
|
||||
|
||||
def _existing_video_output(self, dest: Path) -> Path | None:
|
||||
"""Find a yt-dlp output for `dest` regardless of chosen extension.
|
||||
|
||||
Returns `dest` itself if present, else any sibling sharing the same
|
||||
stem (the remuxed container). None if nothing matched.
|
||||
"""
|
||||
dest = Path(dest)
|
||||
if dest.exists():
|
||||
return dest
|
||||
stem = dest.with_suffix("").name
|
||||
parent = dest.parent
|
||||
if not parent.is_dir():
|
||||
return None
|
||||
for cand in sorted(parent.iterdir()):
|
||||
if cand.is_file() and cand.stem == stem and cand.name != dest.name:
|
||||
return cand
|
||||
return None
|
||||
|
||||
# -- validation --------------------------------------------------------
|
||||
|
||||
def _validate_path(self, path: Path, artist_slug: str) -> str | None:
|
||||
"""Validate a freshly-written file; quarantine + return reason if bad.
|
||||
|
||||
Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown
|
||||
formats, move the corrupt file to _quarantine/<slug>/patreon, and return
|
||||
the failure reason string (None when ok / not validatable / disabled).
|
||||
"""
|
||||
if not self._validate or not is_validatable(path):
|
||||
return None
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
return None
|
||||
if result.ok:
|
||||
return None
|
||||
quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon"
|
||||
try:
|
||||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||||
dest = quarantine_root / path.name
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
|
||||
counter += 1
|
||||
shutil.move(str(path), str(dest))
|
||||
except OSError as exc:
|
||||
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
|
||||
return result.reason or "validation failed"
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
def _write_sidecar(self, post: dict, media_path: Path) -> Path:
|
||||
"""Write the importer-consumed sidecar next to `media_path`.
|
||||
|
||||
Patreon uses base-default sidecar keys (parse_sidecar maps
|
||||
category->platform, id->external_post_id, title->post_title,
|
||||
content->description, published_at->post_date, url->post_url). Patreon
|
||||
registers no derive_post_url, so `url` is trusted as the permalink — we
|
||||
pass the post's attributes.url.
|
||||
"""
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
content = attrs.get("content")
|
||||
published = attrs.get("published_at")
|
||||
url = attrs.get("url")
|
||||
data = {
|
||||
"category": "patreon",
|
||||
"id": str(post.get("id") or ""),
|
||||
"title": title if isinstance(title, str) else "",
|
||||
"content": content if isinstance(content, str) else "",
|
||||
"published_at": published if isinstance(published, str) else None,
|
||||
"url": url if isinstance(url, str) else None,
|
||||
}
|
||||
sidecar_path = media_path.with_suffix(".json")
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
Reference in New Issue
Block a user