refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was importing patreon privates — wrong owner). New backend/app/services/ native_ingest_common.py is the neutral home for: - make_session (was _load_session ×2), retry_after_seconds + 429 constants, sanitize_segment, basename_from_url, post_dir_name, MediaOutcome / PostRecordOutcome. - BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it; each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone. - Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only). post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO). Tests repointed to the single source at every consumer (rule 93 / §8b parity): patreon_client/downloader, subscribestar_native. Exception-trio consolidation + base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -26,9 +26,7 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookiejar
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
@@ -39,46 +37,20 @@ from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import filehash_from_url, safe_ext
|
||||
from ..utils.paths import filehash_from_url
|
||||
from ..utils.prosemirror import post_body_html
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
basename_from_url,
|
||||
make_session,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_POSTS_URL = "https://www.patreon.com/api/posts"
|
||||
_USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# 429 backoff (plan #703): ride out a transient API rate-limit instead of
|
||||
# failing the whole walk (which would stamp RATE_LIMITED → platform-wide
|
||||
# cooldown → every Patreon source dark). Honor the server's `Retry-After`;
|
||||
# otherwise exponential base·2^(n-1), capped. Only after the retries are
|
||||
# exhausted does the 429 propagate as terminal RATE_LIMITED.
|
||||
_MAX_429_RETRIES = 3
|
||||
_BACKOFF_BASE_SECONDS = 2.0
|
||||
_BACKOFF_CAP_SECONDS = 30.0
|
||||
|
||||
|
||||
def _retry_after_seconds(
|
||||
resp: requests.Response,
|
||||
attempt: int,
|
||||
*,
|
||||
base: float = _BACKOFF_BASE_SECONDS,
|
||||
cap: float = _BACKOFF_CAP_SECONDS,
|
||||
) -> float:
|
||||
"""Backoff delay for a 429: the `Retry-After` seconds header if present and
|
||||
numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form
|
||||
of Retry-After is rare here and falls through to exponential.)"""
|
||||
header = resp.headers.get("Retry-After")
|
||||
if header:
|
||||
try:
|
||||
return min(float(header), cap)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return min(base * (2 ** max(0, attempt - 1)), cap)
|
||||
|
||||
# JSON:API request contract (observed from real traffic — see module plan).
|
||||
_INCLUDE = (
|
||||
"campaign,access_rules,attachments,attachments_media,audio,images,media,"
|
||||
@@ -168,49 +140,12 @@ class MediaItem:
|
||||
post_id: str
|
||||
|
||||
|
||||
def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": _USER_AGENT,
|
||||
"Accept": "application/vnd.api+json",
|
||||
}
|
||||
)
|
||||
if cookies_path and os.path.isfile(str(cookies_path)):
|
||||
try:
|
||||
jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
session.cookies = jar # type: ignore[assignment]
|
||||
except (OSError, http.cookiejar.LoadError) as exc:
|
||||
log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc)
|
||||
return session
|
||||
|
||||
|
||||
def _filehash(url: str) -> str | None:
|
||||
# Delegate to the shared extractor (utils.paths) so capture-time persistence
|
||||
# and render-time inline-image matching use the EXACT same identity.
|
||||
return filehash_from_url(url)
|
||||
|
||||
|
||||
def _basename_from_url(url: str) -> str:
|
||||
"""Derive a sane filename from a URL when the media has no file_name.
|
||||
|
||||
Strips query/fragment, takes the path basename, and drops a junk
|
||||
extension (the importer._safe_ext gotcha) so we never write base64 noise
|
||||
as a name. Falls back to the filehash, then to "file".
|
||||
"""
|
||||
path = urlsplit(url).path
|
||||
base = os.path.basename(path)
|
||||
if base:
|
||||
ext = safe_ext(base)
|
||||
stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
|
||||
# Keep the stem bounded; URL-encoded stems can be enormous.
|
||||
stem = stem[:120] or "file"
|
||||
return f"{stem}{ext}"
|
||||
fh = _filehash(url)
|
||||
return fh or "file"
|
||||
|
||||
|
||||
def parse_cursor_from_url(url: str | None) -> str | None:
|
||||
"""Extract the `page[cursor]` query param from a links.next URL."""
|
||||
if not url:
|
||||
@@ -238,7 +173,7 @@ class PatreonClient:
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
):
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._session = _load_session(cookies_path)
|
||||
self._session = make_session(cookies_path, accept="application/vnd.api+json")
|
||||
# Politeness: seconds to sleep before each /api/posts page fetch (paces
|
||||
# the rate-limited API endpoint). 0 = no pacing. plan #703.
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
@@ -283,7 +218,7 @@ class PatreonClient:
|
||||
# through to the terminal RATE_LIMITED raise below.
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = _retry_after_seconds(resp, attempt)
|
||||
delay = retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)",
|
||||
campaign_id, delay, attempt, self._max_retries,
|
||||
@@ -416,7 +351,7 @@ class PatreonClient:
|
||||
# uses. A genuine schema change shows up as no URL (above) or a media id
|
||||
# absent from `included` (caller), not a missing name.
|
||||
file_name = attrs.get("file_name")
|
||||
filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url)
|
||||
filename = file_name if isinstance(file_name, str) and file_name else basename_from_url(url)
|
||||
return MediaItem(
|
||||
url=url,
|
||||
filename=filename,
|
||||
@@ -462,7 +397,7 @@ class PatreonClient:
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=large_url,
|
||||
filename=_basename_from_url(large_url),
|
||||
filename=basename_from_url(large_url),
|
||||
kind="image_large",
|
||||
filehash=_filehash(large_url),
|
||||
post_id=post_id,
|
||||
@@ -481,7 +416,7 @@ class PatreonClient:
|
||||
filename = (
|
||||
pf_name
|
||||
if isinstance(pf_name, str) and pf_name
|
||||
else _basename_from_url(pf_url)
|
||||
else basename_from_url(pf_url)
|
||||
)
|
||||
items.append(
|
||||
MediaItem(
|
||||
@@ -503,7 +438,7 @@ class PatreonClient:
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=src,
|
||||
filename=_basename_from_url(src),
|
||||
filename=basename_from_url(src),
|
||||
kind="content",
|
||||
filehash=_filehash(src),
|
||||
post_id=post_id,
|
||||
|
||||
Reference in New Issue
Block a user