refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was importing patreon privates — wrong owner). New backend/app/services/ native_ingest_common.py is the neutral home for: - make_session (was _load_session ×2), retry_after_seconds + 429 constants, sanitize_segment, basename_from_url, post_dir_name, MediaOutcome / PostRecordOutcome. - BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it; each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone. - Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only). post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO). Tests repointed to the single source at every consumer (rule 93 / §8b parity): patreon_client/downloader, subscribestar_native. Exception-trio consolidation + base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,310 @@
|
||||
"""Shared primitives for the native-ingest platform adapters (Patreon,
|
||||
SubscribeStar, …) — the single home for logic the per-platform client/downloader
|
||||
modules would otherwise each copy.
|
||||
|
||||
DRY pass 2026-06-17 (#899): these used to live in `patreon_*` with the
|
||||
SubscribeStar modules importing patreon privates (wrong owner + sibling-coupling).
|
||||
They're platform-agnostic, so they live here and both adapters import them. The
|
||||
per-platform modules keep only what genuinely differs (feed parsing, the media
|
||||
shape, Patreon's Mux/yt-dlp video branch + detail-fetch enrichment).
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import http.cookiejar
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import filehash_from_url, safe_ext
|
||||
from .file_validator import is_validatable, quarantine_file, validate_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# 429 backoff (plan #703): ride out a transient API rate-limit instead of failing
|
||||
# the whole walk. Honor the server's Retry-After; else exponential, capped.
|
||||
_MAX_429_RETRIES = 3
|
||||
_BACKOFF_BASE_SECONDS = 2.0
|
||||
_BACKOFF_CAP_SECONDS = 30.0
|
||||
|
||||
# Media-download tuning (shared by every platform downloader).
|
||||
_TIMEOUT_SECONDS = 120.0
|
||||
_CHUNK = 1 << 16
|
||||
_MAX_MEDIA_RETRIES = 3
|
||||
_TRANSIENT_TRANSPORT_EXC = (
|
||||
requests.ConnectionError,
|
||||
requests.Timeout,
|
||||
requests.exceptions.ChunkedEncodingError,
|
||||
)
|
||||
|
||||
_TITLE_MAX = 40
|
||||
# Windows/gallery-dl path-restrict forbidden set + path separators.
|
||||
_FORBIDDEN = set('<>:"/\\|?*')
|
||||
|
||||
|
||||
# -- HTTP session ----------------------------------------------------------
|
||||
|
||||
def make_session(
|
||||
cookies_path: str | Path | None,
|
||||
*,
|
||||
accept: str = "*/*",
|
||||
extra_headers: dict | None = None,
|
||||
) -> requests.Session:
|
||||
"""Build a requests.Session loaded with the Netscape cookies.txt
|
||||
CredentialService materializes. `accept` sets the Accept header (the JSON:API
|
||||
vs HTML feed differ); `extra_headers` adds platform headers (e.g.
|
||||
X-Requested-With). Missing/unparseable cookies log a warning, never fail."""
|
||||
session = requests.Session()
|
||||
headers = {"User-Agent": _USER_AGENT, "Accept": accept}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
session.headers.update(headers)
|
||||
if cookies_path and os.path.isfile(str(cookies_path)):
|
||||
try:
|
||||
jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
session.cookies = jar # type: ignore[assignment]
|
||||
except (OSError, http.cookiejar.LoadError) as exc:
|
||||
log.warning("Could not load cookies from %s: %s", cookies_path, exc)
|
||||
return session
|
||||
|
||||
|
||||
def retry_after_seconds(
|
||||
resp: requests.Response,
|
||||
attempt: int,
|
||||
*,
|
||||
base: float = _BACKOFF_BASE_SECONDS,
|
||||
cap: float = _BACKOFF_CAP_SECONDS,
|
||||
) -> float:
|
||||
"""Backoff for a 429: the numeric Retry-After header if present, else
|
||||
exponential base·2^(attempt-1), both capped."""
|
||||
header = resp.headers.get("Retry-After")
|
||||
if header:
|
||||
try:
|
||||
return min(float(header), cap)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return min(base * (2 ** max(0, attempt - 1)), cap)
|
||||
|
||||
|
||||
# -- filename / path helpers -----------------------------------------------
|
||||
|
||||
def sanitize_segment(name: str) -> str:
|
||||
"""Make `name` safe for one filesystem path segment: replace separators, the
|
||||
Windows-forbidden set, and control chars with `_`; strip trailing dots/spaces
|
||||
(gallery-dl path-restrict). Never empty (falls back to `_`)."""
|
||||
out = ["_" if (ch in _FORBIDDEN or ord(ch) < 32) else ch for ch in name]
|
||||
cleaned = "".join(out).rstrip(". ")
|
||||
return cleaned or "_"
|
||||
|
||||
|
||||
def basename_from_url(url: str) -> str:
|
||||
"""Derive a sane filename from a URL when the media has no name: path basename
|
||||
with a junk-extension guard (safe_ext), bounded stem; falls back to the URL's
|
||||
content hash, then "file"."""
|
||||
path = urlsplit(url).path
|
||||
base = os.path.basename(path)
|
||||
if base:
|
||||
ext = safe_ext(base)
|
||||
stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
|
||||
stem = stem[:120] or "file"
|
||||
return f"{stem}{ext}"
|
||||
return filehash_from_url(url) or "file"
|
||||
|
||||
|
||||
def post_dir_name(post: dict) -> str:
|
||||
"""`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's layout (date prefix
|
||||
omitted when published_at is missing/unparseable; title is empty for platforms
|
||||
with no title field). Accepts both ISO and trailing-`Z` published_at."""
|
||||
post_id = str(post.get("id") or "")
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
|
||||
published = attrs.get("published_at")
|
||||
date_prefix = None
|
||||
if isinstance(published, str) and published:
|
||||
s = published.strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
try:
|
||||
date_prefix = f"{datetime.fromisoformat(s):%Y-%m-%d}"
|
||||
except ValueError:
|
||||
date_prefix = None
|
||||
raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
|
||||
return sanitize_segment(raw)
|
||||
|
||||
|
||||
# -- per-item download outcomes (shared dataclasses) -----------------------
|
||||
|
||||
@dataclass
|
||||
class MediaOutcome:
|
||||
"""Per-media result of a download_post pass. status ∈ downloaded /
|
||||
skipped_seen / skipped_disk / quarantined / error. `path` is the on-disk file
|
||||
(downloaded / skipped_disk), the quarantine dest (quarantined), or None;
|
||||
`error` is the failure/validation reason (error/quarantined) else None."""
|
||||
|
||||
media: object
|
||||
status: str
|
||||
path: Path | None
|
||||
error: str | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PostRecordOutcome:
|
||||
"""Result of write_post_record — mirrors the MediaOutcome contract so the core
|
||||
reports per-post handling. `path` is the _post.json sidecar (None when the post
|
||||
had no id); the rest is the captured body's shape for the run log."""
|
||||
|
||||
path: Path | None
|
||||
post_type: str | None
|
||||
title: str | None
|
||||
body_chars: int
|
||||
|
||||
|
||||
# -- base downloader (shared fetch/validate plumbing) ----------------------
|
||||
|
||||
class BaseNativeDownloader:
|
||||
"""Shared download plumbing for native-platform downloaders: the streaming
|
||||
GET (transient-retry + Range-resume) and file validation/quarantine. Platform
|
||||
downloaders subclass this and implement `download_post` / `write_post_record`
|
||||
/ the per-media sidecar (and any platform-specific fetch, e.g. Patreon's
|
||||
Mux/yt-dlp video branch). PURE: no DB; the seen-skip is an injected predicate.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
platform: str,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self.platform = platform
|
||||
self._validate = validate
|
||||
self._rate_limit = rate_limit or 0.0
|
||||
self.session = session if session is not None else make_session(cookies_path)
|
||||
|
||||
# -- download seams ----------------------------------------------------
|
||||
|
||||
def _fetch_get(self, url: str, dest: Path) -> Path:
|
||||
"""Stream `url` to a .part then atomic-rename to `dest`."""
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
try:
|
||||
self._fetch_to_file(url, part)
|
||||
except Exception:
|
||||
with contextlib.suppress(OSError):
|
||||
part.unlink()
|
||||
raise
|
||||
os.replace(part, dest)
|
||||
return dest
|
||||
|
||||
def _fetch_to_file(self, url: str, dest: Path) -> None:
|
||||
"""Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
|
||||
429 honoring Retry-After, 5xx) with backoff + resume-from-disk (Range);
|
||||
failing fast on permanent 4xx (404/403). Resume: a retry with bytes on
|
||||
disk asks `Range: bytes=<have>-`; 206 → append, 200 → restart clean, 416 →
|
||||
already complete. The caller stages into a `.part` so a non-range server
|
||||
never corrupts the output."""
|
||||
attempt = 0
|
||||
while True:
|
||||
have = dest.stat().st_size if dest.exists() else 0
|
||||
headers = {"Range": f"bytes={have}-"} if have > 0 else None
|
||||
try:
|
||||
resp = self.session.get(
|
||||
url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
|
||||
)
|
||||
if (resp.status_code == 429 or resp.status_code >= 500) \
|
||||
and attempt < _MAX_MEDIA_RETRIES:
|
||||
attempt += 1
|
||||
delay = retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"%s media transient HTTP %d (%s) — backing off %.1fs "
|
||||
"(retry %d/%d)",
|
||||
self.platform, resp.status_code, url, delay, attempt,
|
||||
_MAX_MEDIA_RETRIES,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if have > 0 and resp.status_code == 416:
|
||||
return
|
||||
resp.raise_for_status()
|
||||
mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
|
||||
with open(dest, mode) as fh:
|
||||
for chunk in resp.iter_content(chunk_size=_CHUNK):
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
return
|
||||
except _TRANSIENT_TRANSPORT_EXC as exc:
|
||||
if attempt >= _MAX_MEDIA_RETRIES:
|
||||
raise
|
||||
attempt += 1
|
||||
delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
|
||||
log.warning(
|
||||
"%s media transport error (%s) — backing off %.1fs "
|
||||
"(retry %d/%d): %s",
|
||||
self.platform, url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
# -- validation --------------------------------------------------------
|
||||
|
||||
def _validate_path(
|
||||
self, path: Path, artist_slug: str, source_url: str | None = None
|
||||
) -> tuple[str | None, Path | None]:
|
||||
"""Validate a freshly-written file; quarantine if bad (shared
|
||||
file_validator move + provenance sidecar). Returns (reason,
|
||||
quarantine_dest) when quarantined, else (None, None). Logs the quarantine
|
||||
so a corrupt file is visible in the worker logs, not just counted (#899
|
||||
L2)."""
|
||||
if not self._validate or not is_validatable(path):
|
||||
return None, None
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
return None, None
|
||||
if result.ok:
|
||||
return None, None
|
||||
dest = quarantine_file(
|
||||
self.images_root, path, artist_slug, self.platform,
|
||||
url=source_url, result=result,
|
||||
)
|
||||
reason = result.reason or "validation failed"
|
||||
log.warning(
|
||||
"%s quarantined %s (%s) — %s",
|
||||
self.platform, dest or path, artist_slug, reason,
|
||||
)
|
||||
return reason, (dest or path)
|
||||
|
||||
# -- sidecar (per-media, minimal) --------------------------------------
|
||||
|
||||
def _write_minimal_sidecar(
|
||||
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||
) -> Path:
|
||||
"""Post-first per-media sidecar (#856): image identity ONLY
|
||||
(category/id/source_url). The post body/links live solely in _post.json."""
|
||||
data: dict = {"category": self.platform, "id": str(post.get("id") or "")}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path = media_path.with_suffix(".json")
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
Reference in New Issue
Block a user