"""Native SubscribeStar media downloader — the SubscribeStar counterpart to patreon_downloader. Given a SubscribeStar post and its resolved `MediaItem`s (subscribestar_client.extract_media), download the media to gallery-dl's on-disk layout (so existing gallery-dl downloads are recognized on disk and not re-fetched at cutover), write the post-first sidecars the importer consumes, and report per-media outcomes. Simpler than the Patreon downloader: SubscribeStar serves every upload as a direct file via `/post_uploads?payload=...` (plain GET — no Mux/HLS, no yt-dlp), and the full post body is already present in the feed HTML (no detail-endpoint enrichment). PURE: no DB; the seen-skip is an injected predicate. On-disk layout (matches gallery-dl's subscribestar config `{date:%Y-%m-%d}_{id}_{title[:40]}` / `{num:>02}_{filename}`): SubscribeStar posts have no title, so the directory is `__` — the display title is synthesized by the importer from the body, so the bare dir name is on-disk only. FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ from __future__ import annotations import json import logging import time from collections.abc import Callable from pathlib import Path import requests from .native_ingest_common import ( BaseNativeDownloader, MediaOutcome, PostRecordOutcome, post_dir_name, sanitize_segment, ) log = logging.getLogger(__name__) class SubscribeStarDownloader(BaseNativeDownloader): """Download resolved SubscribeStar media to gallery-dl's on-disk layout. Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. No video branch (SubscribeStar serves files directly via /post_uploads) and no detail-fetch (the body is already in the feed HTML). PURE: no DB.""" def __init__( self, images_root: Path, cookies_path: str | None = None, *, validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, ): super().__init__( images_root, cookies_path, platform="subscribestar", validate=validate, rate_limit=rate_limit, session=session, ) # -- public ------------------------------------------------------------ def download_post( self, post: dict, media_items: list, artist_slug: str, *, is_seen: Callable[[object], bool] = lambda m: False, should_stop: Callable[[], bool] = lambda: False, recapture: bool = False, ) -> list[MediaOutcome]: """Download every media item of one post; return per-item outcomes. Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box, recapture surfacing) minus the video branch.""" post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post) outcomes: list[MediaOutcome] = [] for i, media in enumerate(media_items, start=1): if should_stop(): break try: outcomes.append( self._download_one( post, media, post_dir, artist_slug, i, is_seen, recapture=recapture, ) ) except Exception as exc: # resilient: isolate one item's failure log.warning( "SubscribeStar media failed (post %s, item %d): %s", post.get("id"), i, exc, ) outcomes.append( MediaOutcome(media=media, status="error", path=None, error=str(exc)) ) return outcomes # -- per-item ---------------------------------------------------------- def _download_one( self, post: dict, media, post_dir: Path, artist_slug: str, index: int, is_seen: Callable[[object], bool], *, recapture: bool = False, ) -> MediaOutcome: seen = is_seen(media) if seen and not recapture: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) nn = f"{index:02d}" media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}") if media_path.exists(): # tier-2: already on disk return MediaOutcome( media=media, status="skipped_disk", path=media_path, error=None ) # recapture: a seen item not on disk is NOT re-downloaded (recovery's job). if seen: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) post_dir.mkdir(parents=True, exist_ok=True) if self._rate_limit > 0: time.sleep(self._rate_limit) out_path = self._fetch_get(media.url, media_path) reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url) if reason is not None: return MediaOutcome( media=media, status="quarantined", path=quarantine_dest, error=reason, ) self._write_sidecar(post, out_path, source_url=media.url) return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) # The plain-GET streaming path (_fetch_get / _fetch_to_file) and # _validate_path are inherited from BaseNativeDownloader. # -- sidecar ----------------------------------------------------------- def _write_sidecar( self, post: dict, media_path: Path, *, source_url: str | None = None ) -> Path: """Per-media sidecar — post-first (#856): image identity ONLY (category/id/source_url). The post body/links live solely in _post.json.""" return self._write_sidecar_data( post, media_path.with_suffix(".json"), source_url=source_url, minimal=True, ) def _write_sidecar_data( self, post: dict, sidecar_path: Path, *, source_url: str | None = None, minimal: bool = False, ) -> Path: """Serialize the post's metadata. minimal=True → per-media sidecar (image identity only); else the full post record (body/title/date/url).""" if minimal: data = {"category": "subscribestar", "id": str(post.get("id") or "")} if source_url: data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path attrs = post.get("attributes") or {} content = attrs.get("content") # SubscribeStar synthesizes the post permalink from the id (matches the # platforms/subscribestar.py derive_post_url helper). pid = str(post.get("id") or "") data = { "category": "subscribestar", "id": pid, "post_id": pid, # ensures derive_post_url has its key on the native path "title": attrs.get("title") if isinstance(attrs.get("title"), str) else "", "content": content if isinstance(content, str) else "", "published_at": attrs.get("published_at"), } if source_url: data["source_url"] = source_url sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome: """Write the post-first `_post.json` (body/links/metadata) — the sole writer of the post record on the native path. SubscribeStar's body is already in the feed HTML, so no detail-fetch is needed.""" attrs = post.get("attributes") or {} title = attrs.get("title") if isinstance(attrs.get("title"), str) else None post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None pid = str(post.get("id") or "") if not pid: return PostRecordOutcome( path=None, post_type=post_type, title=title, body_chars=0, ) post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post) post_dir.mkdir(parents=True, exist_ok=True) path = self._write_sidecar_data(post, post_dir / "_post.json") body = attrs.get("content") body_chars = len(body) if isinstance(body, str) else 0 return PostRecordOutcome( path=path, post_type=post_type, title=title, body_chars=body_chars, )