refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 17s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m20s

DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing
the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was
importing patreon privates — wrong owner). New backend/app/services/
native_ingest_common.py is the neutral home for:
- make_session (was _load_session ×2), retry_after_seconds + 429 constants,
  sanitize_segment, basename_from_url, post_dir_name, MediaOutcome /
  PostRecordOutcome.
- BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume)
  and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it;
  each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch
  enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the
  divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone.
- Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only).

post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO).
Tests repointed to the single source at every consumer (rule 93 / §8b parity):
patreon_client/downloader, subscribestar_native. Exception-trio consolidation +
base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-17 11:32:28 -04:00
parent 817a002c2b
commit 7ac5c7e522
8 changed files with 402 additions and 504 deletions
+19 -130
View File
@@ -22,62 +22,31 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
from __future__ import annotations
import contextlib
import json
import logging
import os
import time
from collections.abc import Callable
from datetime import datetime
from pathlib import Path
import requests
from .file_validator import is_validatable, quarantine_file, validate_file
from .patreon_downloader import (
from .native_ingest_common import (
BaseNativeDownloader,
MediaOutcome,
PostRecordOutcome,
_sanitize,
post_dir_name,
sanitize_segment,
)
from .subscribestar_client import _MAX_429_RETRIES, _load_session, _retry_after_seconds
log = logging.getLogger(__name__)
_TITLE_MAX = 40
_TIMEOUT_SECONDS = 120.0
_CHUNK = 1 << 16
_MAX_MEDIA_RETRIES = 3
_BACKOFF_CAP_SECONDS = 30.0
_TRANSIENT_TRANSPORT_EXC = (
requests.ConnectionError,
requests.Timeout,
requests.exceptions.ChunkedEncodingError,
)
def _post_dir_name(post: dict) -> str:
"""`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's subscribestar
layout (title is empty for SubscribeStar → trailing underscore). Date prefix
omitted when published_at is missing/unparseable."""
post_id = str(post.get("id") or "")
attrs = post.get("attributes") or {}
title = attrs.get("title")
title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
published = attrs.get("published_at")
date_prefix = None
if isinstance(published, str) and published:
try:
date_prefix = f"{datetime.fromisoformat(published):%Y-%m-%d}"
except ValueError:
date_prefix = None
raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
return _sanitize(raw)
class SubscribeStarDownloader:
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout. PURE:
no DB. The HTTP session is an injectable seam (pass `session=` or monkeypatch
`_fetch_to_file`) so tests run without network."""
class SubscribeStarDownloader(BaseNativeDownloader):
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout.
Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry +
Range-resume) and validation/quarantine. No video branch (SubscribeStar serves
files directly via /post_uploads) and no detail-fetch (the body is already in
the feed HTML). PURE: no DB."""
def __init__(
self,
@@ -87,14 +56,11 @@ class SubscribeStarDownloader:
validate: bool = True,
rate_limit: float = 0.0,
session: requests.Session | None = None,
max_retries: int = _MAX_429_RETRIES,
):
self.images_root = Path(images_root)
self.cookies_path = str(cookies_path) if cookies_path else None
self._validate = validate
self._rate_limit = rate_limit or 0.0
self._max_retries = max_retries
self.session = session if session is not None else _load_session(cookies_path)
super().__init__(
images_root, cookies_path, platform="subscribestar",
validate=validate, rate_limit=rate_limit, session=session,
)
# -- public ------------------------------------------------------------
@@ -111,7 +77,7 @@ class SubscribeStarDownloader:
"""Download every media item of one post; return per-item outcomes.
Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
recapture surfacing) minus the video branch."""
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
outcomes: list[MediaOutcome] = []
for i, media in enumerate(media_items, start=1):
if should_stop():
@@ -151,7 +117,7 @@ class SubscribeStarDownloader:
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
nn = f"{index:02d}"
media_path = post_dir / _sanitize(f"{nn}_{media.filename}")
media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}")
if media_path.exists(): # tier-2: already on disk
return MediaOutcome(
@@ -174,85 +140,8 @@ class SubscribeStarDownloader:
self._write_sidecar(post, out_path, source_url=media.url)
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
# -- download seams ----------------------------------------------------
def _fetch_get(self, url: str, dest: Path) -> Path:
"""Stream `url` to a .part file then atomic-rename to `dest`."""
part = dest.with_name(dest.name + ".part")
try:
self._fetch_to_file(url, part)
except Exception:
with contextlib.suppress(OSError):
part.unlink()
raise
os.replace(part, dest)
return dest
def _fetch_to_file(self, url: str, dest: Path) -> None:
"""Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
429, 5xx) with backoff + resume-from-disk (Range), failing fast on
permanent 4xx. Mirrors PatreonDownloader._fetch_to_file."""
attempt = 0
while True:
have = dest.stat().st_size if dest.exists() else 0
headers = {"Range": f"bytes={have}-"} if have > 0 else None
try:
resp = self.session.get(
url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
)
if (resp.status_code == 429 or resp.status_code >= 500) \
and attempt < _MAX_MEDIA_RETRIES:
attempt += 1
delay = _retry_after_seconds(resp, attempt)
log.warning(
"SubscribeStar media transient HTTP %d (%s) — backing off "
"%.1fs (retry %d/%d)",
resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES,
)
time.sleep(delay)
continue
if have > 0 and resp.status_code == 416:
return # Range past EOF — we already have the whole file
resp.raise_for_status()
mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
with open(dest, mode) as fh:
for chunk in resp.iter_content(chunk_size=_CHUNK):
if chunk:
fh.write(chunk)
return
except _TRANSIENT_TRANSPORT_EXC as exc:
if attempt >= _MAX_MEDIA_RETRIES:
raise
attempt += 1
delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
log.warning(
"SubscribeStar media transport error (%s) — backing off %.1fs "
"(retry %d/%d): %s",
url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
)
time.sleep(delay)
# -- validation --------------------------------------------------------
def _validate_path(
self, path: Path, artist_slug: str, source_url: str | None = None
) -> tuple[str | None, Path | None]:
"""Validate a freshly-written file; quarantine if bad. Mirrors the Patreon
path (shared file_validator)."""
if not self._validate or not is_validatable(path):
return None, None
try:
result = validate_file(path)
except Exception as exc:
log.warning("Validator raised on %s: %s", path, exc)
return None, None
if result.ok:
return None, None
dest = quarantine_file(
self.images_root, path, artist_slug, "subscribestar",
url=source_url, result=result,
)
return (result.reason or "validation failed"), (dest or path)
# The plain-GET streaming path (_fetch_get / _fetch_to_file) and
# _validate_path are inherited from BaseNativeDownloader.
# -- sidecar -----------------------------------------------------------
@@ -307,7 +196,7 @@ class SubscribeStarDownloader:
return PostRecordOutcome(
path=None, post_type=post_type, title=title, body_chars=0,
)
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
post_dir.mkdir(parents=True, exist_ok=True)
path = self._write_sidecar_data(post, post_dir / "_post.json")
body = attrs.get("content")