refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was importing patreon privates — wrong owner). New backend/app/services/ native_ingest_common.py is the neutral home for: - make_session (was _load_session ×2), retry_after_seconds + 429 constants, sanitize_segment, basename_from_url, post_dir_name, MediaOutcome / PostRecordOutcome. - BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it; each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone. - Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only). post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO). Tests repointed to the single source at every consumer (rule 93 / §8b parity): patreon_client/downloader, subscribestar_native. Exception-trio consolidation + base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -22,62 +22,31 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .file_validator import is_validatable, quarantine_file, validate_file
|
||||
from .patreon_downloader import (
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
_sanitize,
|
||||
post_dir_name,
|
||||
sanitize_segment,
|
||||
)
|
||||
from .subscribestar_client import _MAX_429_RETRIES, _load_session, _retry_after_seconds
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TITLE_MAX = 40
|
||||
_TIMEOUT_SECONDS = 120.0
|
||||
_CHUNK = 1 << 16
|
||||
_MAX_MEDIA_RETRIES = 3
|
||||
_BACKOFF_CAP_SECONDS = 30.0
|
||||
_TRANSIENT_TRANSPORT_EXC = (
|
||||
requests.ConnectionError,
|
||||
requests.Timeout,
|
||||
requests.exceptions.ChunkedEncodingError,
|
||||
)
|
||||
|
||||
|
||||
def _post_dir_name(post: dict) -> str:
|
||||
"""`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's subscribestar
|
||||
layout (title is empty for SubscribeStar → trailing underscore). Date prefix
|
||||
omitted when published_at is missing/unparseable."""
|
||||
post_id = str(post.get("id") or "")
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
|
||||
published = attrs.get("published_at")
|
||||
date_prefix = None
|
||||
if isinstance(published, str) and published:
|
||||
try:
|
||||
date_prefix = f"{datetime.fromisoformat(published):%Y-%m-%d}"
|
||||
except ValueError:
|
||||
date_prefix = None
|
||||
raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
|
||||
return _sanitize(raw)
|
||||
|
||||
|
||||
class SubscribeStarDownloader:
|
||||
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout. PURE:
|
||||
no DB. The HTTP session is an injectable seam (pass `session=` or monkeypatch
|
||||
`_fetch_to_file`) so tests run without network."""
|
||||
class SubscribeStarDownloader(BaseNativeDownloader):
|
||||
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout.
|
||||
Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry +
|
||||
Range-resume) and validation/quarantine. No video branch (SubscribeStar serves
|
||||
files directly via /post_uploads) and no detail-fetch (the body is already in
|
||||
the feed HTML). PURE: no DB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -87,14 +56,11 @@ class SubscribeStarDownloader:
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._validate = validate
|
||||
self._rate_limit = rate_limit or 0.0
|
||||
self._max_retries = max_retries
|
||||
self.session = session if session is not None else _load_session(cookies_path)
|
||||
super().__init__(
|
||||
images_root, cookies_path, platform="subscribestar",
|
||||
validate=validate, rate_limit=rate_limit, session=session,
|
||||
)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
@@ -111,7 +77,7 @@ class SubscribeStarDownloader:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
|
||||
recapture surfacing) minus the video branch."""
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for i, media in enumerate(media_items, start=1):
|
||||
if should_stop():
|
||||
@@ -151,7 +117,7 @@ class SubscribeStarDownloader:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
media_path = post_dir / _sanitize(f"{nn}_{media.filename}")
|
||||
media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}")
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
@@ -174,85 +140,8 @@ class SubscribeStarDownloader:
|
||||
self._write_sidecar(post, out_path, source_url=media.url)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- download seams ----------------------------------------------------
|
||||
|
||||
def _fetch_get(self, url: str, dest: Path) -> Path:
|
||||
"""Stream `url` to a .part file then atomic-rename to `dest`."""
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
try:
|
||||
self._fetch_to_file(url, part)
|
||||
except Exception:
|
||||
with contextlib.suppress(OSError):
|
||||
part.unlink()
|
||||
raise
|
||||
os.replace(part, dest)
|
||||
return dest
|
||||
|
||||
def _fetch_to_file(self, url: str, dest: Path) -> None:
|
||||
"""Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
|
||||
429, 5xx) with backoff + resume-from-disk (Range), failing fast on
|
||||
permanent 4xx. Mirrors PatreonDownloader._fetch_to_file."""
|
||||
attempt = 0
|
||||
while True:
|
||||
have = dest.stat().st_size if dest.exists() else 0
|
||||
headers = {"Range": f"bytes={have}-"} if have > 0 else None
|
||||
try:
|
||||
resp = self.session.get(
|
||||
url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
|
||||
)
|
||||
if (resp.status_code == 429 or resp.status_code >= 500) \
|
||||
and attempt < _MAX_MEDIA_RETRIES:
|
||||
attempt += 1
|
||||
delay = _retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"SubscribeStar media transient HTTP %d (%s) — backing off "
|
||||
"%.1fs (retry %d/%d)",
|
||||
resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if have > 0 and resp.status_code == 416:
|
||||
return # Range past EOF — we already have the whole file
|
||||
resp.raise_for_status()
|
||||
mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
|
||||
with open(dest, mode) as fh:
|
||||
for chunk in resp.iter_content(chunk_size=_CHUNK):
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
return
|
||||
except _TRANSIENT_TRANSPORT_EXC as exc:
|
||||
if attempt >= _MAX_MEDIA_RETRIES:
|
||||
raise
|
||||
attempt += 1
|
||||
delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
|
||||
log.warning(
|
||||
"SubscribeStar media transport error (%s) — backing off %.1fs "
|
||||
"(retry %d/%d): %s",
|
||||
url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
# -- validation --------------------------------------------------------
|
||||
|
||||
def _validate_path(
|
||||
self, path: Path, artist_slug: str, source_url: str | None = None
|
||||
) -> tuple[str | None, Path | None]:
|
||||
"""Validate a freshly-written file; quarantine if bad. Mirrors the Patreon
|
||||
path (shared file_validator)."""
|
||||
if not self._validate or not is_validatable(path):
|
||||
return None, None
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
return None, None
|
||||
if result.ok:
|
||||
return None, None
|
||||
dest = quarantine_file(
|
||||
self.images_root, path, artist_slug, "subscribestar",
|
||||
url=source_url, result=result,
|
||||
)
|
||||
return (result.reason or "validation failed"), (dest or path)
|
||||
# The plain-GET streaming path (_fetch_get / _fetch_to_file) and
|
||||
# _validate_path are inherited from BaseNativeDownloader.
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
@@ -307,7 +196,7 @@ class SubscribeStarDownloader:
|
||||
return PostRecordOutcome(
|
||||
path=None, post_type=post_type, title=title, body_chars=0,
|
||||
)
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
body = attrs.get("content")
|
||||
|
||||
Reference in New Issue
Block a user