refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was importing patreon privates — wrong owner). New backend/app/services/ native_ingest_common.py is the neutral home for: - make_session (was _load_session ×2), retry_after_seconds + 429 constants, sanitize_segment, basename_from_url, post_dir_name, MediaOutcome / PostRecordOutcome. - BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it; each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone. - Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only). post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO). Tests repointed to the single source at every consumer (rule 93 / §8b parity): patreon_client/downloader, subscribestar_native. Exception-trio consolidation + base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -27,10 +27,8 @@ plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookiejar
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
@@ -42,16 +40,20 @@ from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import filehash_from_url, safe_ext
|
||||
from .patreon_client import _MAX_429_RETRIES, _retry_after_seconds
|
||||
from ..utils.paths import filehash_from_url
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
basename_from_url,
|
||||
make_session,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
_TIMEOUT_SECONDS = 30.0
|
||||
# Accept header for the HTML feed + JSON "load more" + the XHR marker.
|
||||
_ACCEPT = "text/html,application/xhtml+xml,application/json,*/*"
|
||||
_XHR_HEADERS = {"X-Requested-With": "XMLHttpRequest"}
|
||||
|
||||
# A post block opens with this wrapper; we slice the page between consecutive
|
||||
# occurrences (regex can't match the balanced close of nested divs, so chunk-
|
||||
@@ -122,26 +124,6 @@ class MediaItem:
|
||||
media_id: str
|
||||
|
||||
|
||||
def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": _USER_AGENT,
|
||||
# HTML feed + JSON "load more"; let the server pick.
|
||||
"Accept": "text/html,application/xhtml+xml,application/json,*/*",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
}
|
||||
)
|
||||
if cookies_path and os.path.isfile(str(cookies_path)):
|
||||
try:
|
||||
jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
session.cookies = jar # type: ignore[assignment]
|
||||
except (OSError, http.cookiejar.LoadError) as exc:
|
||||
log.warning("Could not load SubscribeStar cookies from %s: %s", cookies_path, exc)
|
||||
return session
|
||||
|
||||
|
||||
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
|
||||
"""`campaign_id` is the creator URL → (base, slug).
|
||||
|
||||
@@ -167,17 +149,6 @@ def _parse_ss_datetime(text: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _basename_from_url(url: str) -> str:
|
||||
path = urlsplit(url).path
|
||||
base = os.path.basename(path)
|
||||
if base:
|
||||
ext = safe_ext(base)
|
||||
stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
|
||||
stem = stem[:120] or "file"
|
||||
return f"{stem}{ext}"
|
||||
return "file"
|
||||
|
||||
|
||||
class SubscribeStarClient:
|
||||
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
||||
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
||||
@@ -191,7 +162,9 @@ class SubscribeStarClient:
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
):
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._session = _load_session(cookies_path)
|
||||
self._session = make_session(
|
||||
cookies_path, accept=_ACCEPT, extra_headers=_XHR_HEADERS
|
||||
)
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
self._max_retries = max_retries
|
||||
|
||||
@@ -210,7 +183,7 @@ class SubscribeStarClient:
|
||||
) from exc
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = _retry_after_seconds(resp, attempt)
|
||||
delay = retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
|
||||
url, delay, attempt, self._max_retries,
|
||||
@@ -341,7 +314,7 @@ class SubscribeStarClient:
|
||||
url = urljoin(base + "/", rel)
|
||||
media_id = str(it.get("id") or "")
|
||||
name = it.get("original_filename")
|
||||
filename = name if isinstance(name, str) and name else _basename_from_url(url)
|
||||
filename = name if isinstance(name, str) and name else basename_from_url(url)
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=url,
|
||||
|
||||
Reference in New Issue
Block a user