feat(subscribestar): native client + downloader + ingester (post-first) (#890/#891/#892)
Phase-1 steps 2-4 of moving SubscribeStar off gallery-dl onto the native core
ingester. SubscribeStar has no JSON:API, so the client scrapes HTML; the
platform-agnostic core (ingest_core) is unchanged.
- subscribestar_client.py: HTML-scrape read path. iter_posts pages via the
creator page → infinite_scroll-next_page href → JSON {html} fragments
(campaign_id = creator URL; no resolver). extract_media reads the per-post
data-gallery JSON manifest (id/original_filename/type/url). post_record_key,
post_meta, and post_is_gated (best-effort locked-teaser marker, pending a live
locked sample). Loud auth/drift taxonomy (SubscribeStar{API,Auth,Drift}Error).
Parser validated against the real Step-0 fixtures.
- subscribestar_downloader.py: mirrors PatreonDownloader minus the Mux/yt-dlp
branch (SubscribeStar serves files directly via /post_uploads). gallery-dl
on-disk layout so existing downloads dedup on disk at cutover. Post-first:
_post.json owns the body/links; per-media sidecar carries image identity only.
- subscribestar_ingester.py: thin adapter wiring client/downloader/the
SubscribeStar ledgers into the core; ledger_key = filehash else
post_id:media_id; SubscribeStar failure mapping. verify_subscribestar_credential.
- tests: client parsing/pagination/media/gating/record-key/dates, downloader
layout/sidecar/post-record/skip-seen, ingester ledger_key + failure mapping.
Not yet wired into dispatch (Step 5) — these modules are inert until then.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,317 @@
|
||||
"""Native SubscribeStar media downloader — the SubscribeStar counterpart to
|
||||
patreon_downloader.
|
||||
|
||||
Given a SubscribeStar post and its resolved `MediaItem`s
|
||||
(subscribestar_client.extract_media), download the media to gallery-dl's on-disk
|
||||
layout (so existing gallery-dl downloads are recognized on disk and not
|
||||
re-fetched at cutover), write the post-first sidecars the importer consumes, and
|
||||
report per-media outcomes.
|
||||
|
||||
Simpler than the Patreon downloader: SubscribeStar serves every upload as a
|
||||
direct file via `/post_uploads?payload=...` (plain GET — no Mux/HLS, no yt-dlp),
|
||||
and the full post body is already present in the feed HTML (no detail-endpoint
|
||||
enrichment). PURE: no DB; the seen-skip is an injected predicate.
|
||||
|
||||
On-disk layout (matches gallery-dl's subscribestar config
|
||||
`{date:%Y-%m-%d}_{id}_{title[:40]}` / `{num:>02}_{filename}`): SubscribeStar posts
|
||||
have no title, so the directory is `<YYYY-MM-DD>_<post_id>_` — the display title is
|
||||
synthesized by the importer from the body, so the bare dir name is on-disk only.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .file_validator import is_validatable, quarantine_file, validate_file
|
||||
from .patreon_downloader import (
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
_sanitize,
|
||||
)
|
||||
from .subscribestar_client import _MAX_429_RETRIES, _load_session, _retry_after_seconds
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TITLE_MAX = 40
|
||||
_TIMEOUT_SECONDS = 120.0
|
||||
_CHUNK = 1 << 16
|
||||
_MAX_MEDIA_RETRIES = 3
|
||||
_BACKOFF_CAP_SECONDS = 30.0
|
||||
_TRANSIENT_TRANSPORT_EXC = (
|
||||
requests.ConnectionError,
|
||||
requests.Timeout,
|
||||
requests.exceptions.ChunkedEncodingError,
|
||||
)
|
||||
|
||||
|
||||
def _post_dir_name(post: dict) -> str:
|
||||
"""`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's subscribestar
|
||||
layout (title is empty for SubscribeStar → trailing underscore). Date prefix
|
||||
omitted when published_at is missing/unparseable."""
|
||||
post_id = str(post.get("id") or "")
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
|
||||
published = attrs.get("published_at")
|
||||
date_prefix = None
|
||||
if isinstance(published, str) and published:
|
||||
try:
|
||||
date_prefix = f"{datetime.fromisoformat(published):%Y-%m-%d}"
|
||||
except ValueError:
|
||||
date_prefix = None
|
||||
raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
|
||||
return _sanitize(raw)
|
||||
|
||||
|
||||
class SubscribeStarDownloader:
|
||||
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout. PURE:
|
||||
no DB. The HTTP session is an injectable seam (pass `session=` or monkeypatch
|
||||
`_fetch_to_file`) so tests run without network."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._validate = validate
|
||||
self._rate_limit = rate_limit or 0.0
|
||||
self._max_retries = max_retries
|
||||
self.session = session if session is not None else _load_session(cookies_path)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
|
||||
recapture surfacing) minus the video branch."""
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for i, media in enumerate(media_items, start=1):
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(
|
||||
post, media, post_dir, artist_slug, i, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"SubscribeStar media failed (post %s, item %d): %s",
|
||||
post.get("id"), i, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
post_dir: Path,
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
media_path = post_dir / _sanitize(f"{nn}_{media.filename}")
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
)
|
||||
# recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="quarantined", path=quarantine_dest, error=reason,
|
||||
)
|
||||
self._write_sidecar(post, out_path, source_url=media.url)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- download seams ----------------------------------------------------
|
||||
|
||||
def _fetch_get(self, url: str, dest: Path) -> Path:
|
||||
"""Stream `url` to a .part file then atomic-rename to `dest`."""
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
try:
|
||||
self._fetch_to_file(url, part)
|
||||
except Exception:
|
||||
with contextlib.suppress(OSError):
|
||||
part.unlink()
|
||||
raise
|
||||
os.replace(part, dest)
|
||||
return dest
|
||||
|
||||
def _fetch_to_file(self, url: str, dest: Path) -> None:
|
||||
"""Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
|
||||
429, 5xx) with backoff + resume-from-disk (Range), failing fast on
|
||||
permanent 4xx. Mirrors PatreonDownloader._fetch_to_file."""
|
||||
attempt = 0
|
||||
while True:
|
||||
have = dest.stat().st_size if dest.exists() else 0
|
||||
headers = {"Range": f"bytes={have}-"} if have > 0 else None
|
||||
try:
|
||||
resp = self.session.get(
|
||||
url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
|
||||
)
|
||||
if (resp.status_code == 429 or resp.status_code >= 500) \
|
||||
and attempt < _MAX_MEDIA_RETRIES:
|
||||
attempt += 1
|
||||
delay = _retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"SubscribeStar media transient HTTP %d (%s) — backing off "
|
||||
"%.1fs (retry %d/%d)",
|
||||
resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if have > 0 and resp.status_code == 416:
|
||||
return # Range past EOF — we already have the whole file
|
||||
resp.raise_for_status()
|
||||
mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
|
||||
with open(dest, mode) as fh:
|
||||
for chunk in resp.iter_content(chunk_size=_CHUNK):
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
return
|
||||
except _TRANSIENT_TRANSPORT_EXC as exc:
|
||||
if attempt >= _MAX_MEDIA_RETRIES:
|
||||
raise
|
||||
attempt += 1
|
||||
delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
|
||||
log.warning(
|
||||
"SubscribeStar media transport error (%s) — backing off %.1fs "
|
||||
"(retry %d/%d): %s",
|
||||
url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
# -- validation --------------------------------------------------------
|
||||
|
||||
def _validate_path(
|
||||
self, path: Path, artist_slug: str, source_url: str | None = None
|
||||
) -> tuple[str | None, Path | None]:
|
||||
"""Validate a freshly-written file; quarantine if bad. Mirrors the Patreon
|
||||
path (shared file_validator)."""
|
||||
if not self._validate or not is_validatable(path):
|
||||
return None, None
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
return None, None
|
||||
if result.ok:
|
||||
return None, None
|
||||
dest = quarantine_file(
|
||||
self.images_root, path, artist_slug, "subscribestar",
|
||||
url=source_url, result=result,
|
||||
)
|
||||
return (result.reason or "validation failed"), (dest or path)
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
def _write_sidecar(
|
||||
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||
) -> Path:
|
||||
"""Per-media sidecar — post-first (#856): image identity ONLY
|
||||
(category/id/source_url). The post body/links live solely in _post.json."""
|
||||
return self._write_sidecar_data(
|
||||
post, media_path.with_suffix(".json"), source_url=source_url, minimal=True,
|
||||
)
|
||||
|
||||
def _write_sidecar_data(
|
||||
self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
|
||||
minimal: bool = False,
|
||||
) -> Path:
|
||||
"""Serialize the post's metadata. minimal=True → per-media sidecar (image
|
||||
identity only); else the full post record (body/title/date/url)."""
|
||||
if minimal:
|
||||
data = {"category": "subscribestar", "id": str(post.get("id") or "")}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
attrs = post.get("attributes") or {}
|
||||
content = attrs.get("content")
|
||||
# SubscribeStar synthesizes the post permalink from the id (matches the
|
||||
# platforms/subscribestar.py derive_post_url helper).
|
||||
pid = str(post.get("id") or "")
|
||||
data = {
|
||||
"category": "subscribestar",
|
||||
"id": pid,
|
||||
"post_id": pid, # ensures derive_post_url has its key on the native path
|
||||
"title": attrs.get("title") if isinstance(attrs.get("title"), str) else "",
|
||||
"content": content if isinstance(content, str) else "",
|
||||
"published_at": attrs.get("published_at"),
|
||||
}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
|
||||
"""Write the post-first `_post.json` (body/links/metadata) — the sole
|
||||
writer of the post record on the native path. SubscribeStar's body is
|
||||
already in the feed HTML, so no detail-fetch is needed."""
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
|
||||
post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
return PostRecordOutcome(
|
||||
path=None, post_type=post_type, title=title, body_chars=0,
|
||||
)
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
body = attrs.get("content")
|
||||
body_chars = len(body) if isinstance(body, str) else 0
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=post_type, title=title, body_chars=body_chars,
|
||||
)
|
||||
Reference in New Issue
Block a user