"""Shared primitives for the native-ingest platform adapters (Patreon, SubscribeStar, …) — the single home for logic the per-platform client/downloader modules would otherwise each copy. DRY pass 2026-06-17 (#899): these used to live in `patreon_*` with the SubscribeStar modules importing patreon privates (wrong owner + sibling-coupling). They're platform-agnostic, so they live here and both adapters import them. The per-platform modules keep only what genuinely differs (feed parsing, the media shape, Patreon's Mux/yt-dlp video branch + detail-fetch enrichment). FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ from __future__ import annotations import contextlib import http.cookiejar import json import logging import os import time from dataclasses import dataclass from datetime import datetime from pathlib import Path from urllib.parse import urlsplit import requests from ..utils.paths import filehash_from_url, safe_ext from .file_validator import is_validatable, quarantine_file, validate_file log = logging.getLogger(__name__) _USER_AGENT = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" ) # 429 backoff (plan #703): ride out a transient API rate-limit instead of failing # the whole walk. Honor the server's Retry-After; else exponential, capped. _MAX_429_RETRIES = 3 _BACKOFF_BASE_SECONDS = 2.0 _BACKOFF_CAP_SECONDS = 30.0 # Media-download tuning (shared by every platform downloader). _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 _MAX_MEDIA_RETRIES = 3 _TRANSIENT_TRANSPORT_EXC = ( requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError, ) _TITLE_MAX = 40 # Windows/gallery-dl path-restrict forbidden set + path separators. _FORBIDDEN = set('<>:"/\\|?*') # -- HTTP session ---------------------------------------------------------- def make_session( cookies_path: str | Path | None, *, accept: str = "*/*", extra_headers: dict | None = None, ) -> requests.Session: """Build a requests.Session loaded with the Netscape cookies.txt CredentialService materializes. `accept` sets the Accept header (the JSON:API vs HTML feed differ); `extra_headers` adds platform headers (e.g. X-Requested-With). Missing/unparseable cookies log a warning, never fail.""" session = requests.Session() headers = {"User-Agent": _USER_AGENT, "Accept": accept} if extra_headers: headers.update(extra_headers) session.headers.update(headers) if cookies_path and os.path.isfile(str(cookies_path)): try: jar = http.cookiejar.MozillaCookieJar(str(cookies_path)) jar.load(ignore_discard=True, ignore_expires=True) session.cookies = jar # type: ignore[assignment] except (OSError, http.cookiejar.LoadError) as exc: log.warning("Could not load cookies from %s: %s", cookies_path, exc) return session def retry_after_seconds( resp: requests.Response, attempt: int, *, base: float = _BACKOFF_BASE_SECONDS, cap: float = _BACKOFF_CAP_SECONDS, ) -> float: """Backoff for a 429: the numeric Retry-After header if present, else exponential base·2^(attempt-1), both capped.""" header = resp.headers.get("Retry-After") if header: try: return min(float(header), cap) except (TypeError, ValueError): pass return min(base * (2 ** max(0, attempt - 1)), cap) # -- filename / path helpers ----------------------------------------------- def sanitize_segment(name: str) -> str: """Make `name` safe for one filesystem path segment: replace separators, the Windows-forbidden set, and control chars with `_`; strip trailing dots/spaces (gallery-dl path-restrict). Never empty (falls back to `_`).""" out = ["_" if (ch in _FORBIDDEN or ord(ch) < 32) else ch for ch in name] cleaned = "".join(out).rstrip(". ") return cleaned or "_" def basename_from_url(url: str) -> str: """Derive a sane filename from a URL when the media has no name: path basename with a junk-extension guard (safe_ext), bounded stem; falls back to the URL's content hash, then "file".""" path = urlsplit(url).path base = os.path.basename(path) if base: ext = safe_ext(base) stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base stem = stem[:120] or "file" return f"{stem}{ext}" return filehash_from_url(url) or "file" def post_dir_name(post: dict) -> str: """`__` matching gallery-dl's layout (date prefix omitted when published_at is missing/unparseable; title is empty for platforms with no title field). Accepts both ISO and trailing-`Z` published_at.""" post_id = str(post.get("id") or "") attrs = post.get("attributes") or {} title = attrs.get("title") title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX] published = attrs.get("published_at") date_prefix = None if isinstance(published, str) and published: s = published.strip() if s.endswith("Z"): s = s[:-1] + "+00:00" try: date_prefix = f"{datetime.fromisoformat(s):%Y-%m-%d}" except ValueError: date_prefix = None raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}" return sanitize_segment(raw) # -- per-item download outcomes (shared dataclasses) ----------------------- @dataclass class MediaOutcome: """Per-media result of a download_post pass. status ∈ downloaded / skipped_seen / skipped_disk / quarantined / error. `path` is the on-disk file (downloaded / skipped_disk), the quarantine dest (quarantined), or None; `error` is the failure/validation reason (error/quarantined) else None.""" media: object status: str path: Path | None error: str | None @dataclass class PostRecordOutcome: """Result of write_post_record — mirrors the MediaOutcome contract so the core reports per-post handling. `path` is the _post.json sidecar (None when the post had no id); the rest is the captured body's shape for the run log.""" path: Path | None post_type: str | None title: str | None body_chars: int # -- base downloader (shared fetch/validate plumbing) ---------------------- class BaseNativeDownloader: """Shared download plumbing for native-platform downloaders: the streaming GET (transient-retry + Range-resume) and file validation/quarantine. Platform downloaders subclass this and implement `download_post` / `write_post_record` / the per-media sidecar (and any platform-specific fetch, e.g. Patreon's Mux/yt-dlp video branch). PURE: no DB; the seen-skip is an injected predicate. """ def __init__( self, images_root: Path, cookies_path: str | None = None, *, platform: str, validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self.platform = platform self._validate = validate self._rate_limit = rate_limit or 0.0 self.session = session if session is not None else make_session(cookies_path) # -- download seams ---------------------------------------------------- def _fetch_get(self, url: str, dest: Path) -> Path: """Stream `url` to a .part then atomic-rename to `dest`.""" part = dest.with_name(dest.name + ".part") try: self._fetch_to_file(url, part) except Exception: with contextlib.suppress(OSError): part.unlink() raise os.replace(part, dest) return dest def _fetch_to_file(self, url: str, dest: Path) -> None: """Stream a URL to `dest`, retrying TRANSIENT failures (transport blips, 429 honoring Retry-After, 5xx) with backoff + resume-from-disk (Range); failing fast on permanent 4xx (404/403). Resume: a retry with bytes on disk asks `Range: bytes=-`; 206 → append, 200 → restart clean, 416 → already complete. The caller stages into a `.part` so a non-range server never corrupts the output.""" attempt = 0 while True: have = dest.stat().st_size if dest.exists() else 0 headers = {"Range": f"bytes={have}-"} if have > 0 else None try: resp = self.session.get( url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers, ) if (resp.status_code == 429 or resp.status_code >= 500) \ and attempt < _MAX_MEDIA_RETRIES: attempt += 1 delay = retry_after_seconds(resp, attempt) log.warning( "%s media transient HTTP %d (%s) — backing off %.1fs " "(retry %d/%d)", self.platform, resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES, ) time.sleep(delay) continue if have > 0 and resp.status_code == 416: return resp.raise_for_status() mode = "ab" if (have > 0 and resp.status_code == 206) else "wb" with open(dest, mode) as fh: for chunk in resp.iter_content(chunk_size=_CHUNK): if chunk: fh.write(chunk) return except _TRANSIENT_TRANSPORT_EXC as exc: if attempt >= _MAX_MEDIA_RETRIES: raise attempt += 1 delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) log.warning( "%s media transport error (%s) — backing off %.1fs " "(retry %d/%d): %s", self.platform, url, delay, attempt, _MAX_MEDIA_RETRIES, exc, ) time.sleep(delay) # -- validation -------------------------------------------------------- def _validate_path( self, path: Path, artist_slug: str, source_url: str | None = None ) -> tuple[str | None, Path | None]: """Validate a freshly-written file; quarantine if bad (shared file_validator move + provenance sidecar). Returns (reason, quarantine_dest) when quarantined, else (None, None). Logs the quarantine so a corrupt file is visible in the worker logs, not just counted (#899 L2).""" if not self._validate or not is_validatable(path): return None, None try: result = validate_file(path) except Exception as exc: log.warning("Validator raised on %s: %s", path, exc) return None, None if result.ok: return None, None dest = quarantine_file( self.images_root, path, artist_slug, self.platform, url=source_url, result=result, ) reason = result.reason or "validation failed" log.warning( "%s quarantined %s (%s) — %s", self.platform, dest or path, artist_slug, reason, ) return reason, (dest or path) # -- sidecar (per-media, minimal) -------------------------------------- def _write_minimal_sidecar( self, post: dict, media_path: Path, *, source_url: str | None = None ) -> Path: """Post-first per-media sidecar (#856): image identity ONLY (category/id/source_url). The post body/links live solely in _post.json.""" data: dict = {"category": self.platform, "id": str(post.get("id") or "")} if source_url: data["source_url"] = source_url sidecar_path = media_path.with_suffix(".json") sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path