"""Native SubscribeStar media downloader — the SubscribeStar counterpart to patreon_downloader. Given a SubscribeStar post and its resolved `MediaItem`s (subscribestar_client.extract_media), download the media to gallery-dl's on-disk layout (so existing gallery-dl downloads are recognized on disk and not re-fetched at cutover), write the post-first sidecars the importer consumes, and report per-media outcomes. Simpler than the Patreon downloader: SubscribeStar serves every upload as a direct file via `/post_uploads?payload=...` (plain GET — no Mux/HLS, no yt-dlp), and the full post body is already present in the feed HTML (no detail-endpoint enrichment). PURE: no DB; the seen-skip is an injected predicate. On-disk layout (matches gallery-dl's subscribestar config `{date:%Y-%m-%d}_{id}_{title[:40]}` / `{num:>02}_{filename}`): SubscribeStar posts have no title, so the directory is `__` — the display title is synthesized by the importer from the body, so the bare dir name is on-disk only. FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ from __future__ import annotations import contextlib import json import logging import os import time from collections.abc import Callable from datetime import datetime from pathlib import Path import requests from .file_validator import is_validatable, quarantine_file, validate_file from .patreon_downloader import ( MediaOutcome, PostRecordOutcome, _sanitize, ) from .subscribestar_client import _MAX_429_RETRIES, _load_session, _retry_after_seconds log = logging.getLogger(__name__) _TITLE_MAX = 40 _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 _MAX_MEDIA_RETRIES = 3 _BACKOFF_CAP_SECONDS = 30.0 _TRANSIENT_TRANSPORT_EXC = ( requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError, ) def _post_dir_name(post: dict) -> str: """`__` matching gallery-dl's subscribestar layout (title is empty for SubscribeStar → trailing underscore). Date prefix omitted when published_at is missing/unparseable.""" post_id = str(post.get("id") or "") attrs = post.get("attributes") or {} title = attrs.get("title") title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX] published = attrs.get("published_at") date_prefix = None if isinstance(published, str) and published: try: date_prefix = f"{datetime.fromisoformat(published):%Y-%m-%d}" except ValueError: date_prefix = None raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}" return _sanitize(raw) class SubscribeStarDownloader: """Download resolved SubscribeStar media to gallery-dl's on-disk layout. PURE: no DB. The HTTP session is an injectable seam (pass `session=` or monkeypatch `_fetch_to_file`) so tests run without network.""" def __init__( self, images_root: Path, cookies_path: str | None = None, *, validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, max_retries: int = _MAX_429_RETRIES, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self._validate = validate self._rate_limit = rate_limit or 0.0 self._max_retries = max_retries self.session = session if session is not None else _load_session(cookies_path) # -- public ------------------------------------------------------------ def download_post( self, post: dict, media_items: list, artist_slug: str, *, is_seen: Callable[[object], bool] = lambda m: False, should_stop: Callable[[], bool] = lambda: False, recapture: bool = False, ) -> list[MediaOutcome]: """Download every media item of one post; return per-item outcomes. Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box, recapture surfacing) minus the video branch.""" post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post) outcomes: list[MediaOutcome] = [] for i, media in enumerate(media_items, start=1): if should_stop(): break try: outcomes.append( self._download_one( post, media, post_dir, artist_slug, i, is_seen, recapture=recapture, ) ) except Exception as exc: # resilient: isolate one item's failure log.warning( "SubscribeStar media failed (post %s, item %d): %s", post.get("id"), i, exc, ) outcomes.append( MediaOutcome(media=media, status="error", path=None, error=str(exc)) ) return outcomes # -- per-item ---------------------------------------------------------- def _download_one( self, post: dict, media, post_dir: Path, artist_slug: str, index: int, is_seen: Callable[[object], bool], *, recapture: bool = False, ) -> MediaOutcome: seen = is_seen(media) if seen and not recapture: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) nn = f"{index:02d}" media_path = post_dir / _sanitize(f"{nn}_{media.filename}") if media_path.exists(): # tier-2: already on disk return MediaOutcome( media=media, status="skipped_disk", path=media_path, error=None ) # recapture: a seen item not on disk is NOT re-downloaded (recovery's job). if seen: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) post_dir.mkdir(parents=True, exist_ok=True) if self._rate_limit > 0: time.sleep(self._rate_limit) out_path = self._fetch_get(media.url, media_path) reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url) if reason is not None: return MediaOutcome( media=media, status="quarantined", path=quarantine_dest, error=reason, ) self._write_sidecar(post, out_path, source_url=media.url) return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) # -- download seams ---------------------------------------------------- def _fetch_get(self, url: str, dest: Path) -> Path: """Stream `url` to a .part file then atomic-rename to `dest`.""" part = dest.with_name(dest.name + ".part") try: self._fetch_to_file(url, part) except Exception: with contextlib.suppress(OSError): part.unlink() raise os.replace(part, dest) return dest def _fetch_to_file(self, url: str, dest: Path) -> None: """Stream a URL to `dest`, retrying TRANSIENT failures (transport blips, 429, 5xx) with backoff + resume-from-disk (Range), failing fast on permanent 4xx. Mirrors PatreonDownloader._fetch_to_file.""" attempt = 0 while True: have = dest.stat().st_size if dest.exists() else 0 headers = {"Range": f"bytes={have}-"} if have > 0 else None try: resp = self.session.get( url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers, ) if (resp.status_code == 429 or resp.status_code >= 500) \ and attempt < _MAX_MEDIA_RETRIES: attempt += 1 delay = _retry_after_seconds(resp, attempt) log.warning( "SubscribeStar media transient HTTP %d (%s) — backing off " "%.1fs (retry %d/%d)", resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES, ) time.sleep(delay) continue if have > 0 and resp.status_code == 416: return # Range past EOF — we already have the whole file resp.raise_for_status() mode = "ab" if (have > 0 and resp.status_code == 206) else "wb" with open(dest, mode) as fh: for chunk in resp.iter_content(chunk_size=_CHUNK): if chunk: fh.write(chunk) return except _TRANSIENT_TRANSPORT_EXC as exc: if attempt >= _MAX_MEDIA_RETRIES: raise attempt += 1 delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) log.warning( "SubscribeStar media transport error (%s) — backing off %.1fs " "(retry %d/%d): %s", url, delay, attempt, _MAX_MEDIA_RETRIES, exc, ) time.sleep(delay) # -- validation -------------------------------------------------------- def _validate_path( self, path: Path, artist_slug: str, source_url: str | None = None ) -> tuple[str | None, Path | None]: """Validate a freshly-written file; quarantine if bad. Mirrors the Patreon path (shared file_validator).""" if not self._validate or not is_validatable(path): return None, None try: result = validate_file(path) except Exception as exc: log.warning("Validator raised on %s: %s", path, exc) return None, None if result.ok: return None, None dest = quarantine_file( self.images_root, path, artist_slug, "subscribestar", url=source_url, result=result, ) return (result.reason or "validation failed"), (dest or path) # -- sidecar ----------------------------------------------------------- def _write_sidecar( self, post: dict, media_path: Path, *, source_url: str | None = None ) -> Path: """Per-media sidecar — post-first (#856): image identity ONLY (category/id/source_url). The post body/links live solely in _post.json.""" return self._write_sidecar_data( post, media_path.with_suffix(".json"), source_url=source_url, minimal=True, ) def _write_sidecar_data( self, post: dict, sidecar_path: Path, *, source_url: str | None = None, minimal: bool = False, ) -> Path: """Serialize the post's metadata. minimal=True → per-media sidecar (image identity only); else the full post record (body/title/date/url).""" if minimal: data = {"category": "subscribestar", "id": str(post.get("id") or "")} if source_url: data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path attrs = post.get("attributes") or {} content = attrs.get("content") # SubscribeStar synthesizes the post permalink from the id (matches the # platforms/subscribestar.py derive_post_url helper). pid = str(post.get("id") or "") data = { "category": "subscribestar", "id": pid, "post_id": pid, # ensures derive_post_url has its key on the native path "title": attrs.get("title") if isinstance(attrs.get("title"), str) else "", "content": content if isinstance(content, str) else "", "published_at": attrs.get("published_at"), } if source_url: data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome: """Write the post-first `_post.json` (body/links/metadata) — the sole writer of the post record on the native path. SubscribeStar's body is already in the feed HTML, so no detail-fetch is needed.""" attrs = post.get("attributes") or {} title = attrs.get("title") if isinstance(attrs.get("title"), str) else None post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None pid = str(post.get("id") or "") if not pid: return PostRecordOutcome( path=None, post_type=post_type, title=title, body_chars=0, ) post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post) post_dir.mkdir(parents=True, exist_ok=True) path = self._write_sidecar_data(post, post_dir / "_post.json") body = attrs.get("content") body_chars = len(body) if isinstance(body, str) else 0 return PostRecordOutcome( path=path, post_type=post_type, title=title, body_chars=body_chars, )