refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 17s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m20s

DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing
the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was
importing patreon privates — wrong owner). New backend/app/services/
native_ingest_common.py is the neutral home for:
- make_session (was _load_session ×2), retry_after_seconds + 429 constants,
  sanitize_segment, basename_from_url, post_dir_name, MediaOutcome /
  PostRecordOutcome.
- BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume)
  and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it;
  each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch
  enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the
  divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone.
- Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only).

post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO).
Tests repointed to the single source at every consumer (rule 93 / §8b parity):
patreon_client/downloader, subscribestar_native. Exception-trio consolidation +
base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-17 11:32:28 -04:00
parent 817a002c2b
commit 7ac5c7e522
8 changed files with 402 additions and 504 deletions
+7 -7
View File
@@ -12,13 +12,13 @@ import pytest
import requests
from backend.app.services import patreon_client as pc_mod
from backend.app.services.native_ingest_common import retry_after_seconds
from backend.app.services.patreon_client import (
MediaItem,
PatreonAPIError,
PatreonAuthError,
PatreonClient,
PatreonDriftError,
_retry_after_seconds,
parse_cursor_from_url,
)
@@ -288,19 +288,19 @@ def test_verify_auth_inconclusive_on_network(monkeypatch):
def test_retry_after_seconds_honors_numeric_header():
assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0
assert retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0
def test_retry_after_seconds_exponential_without_header():
resp = _FakeResp(429)
assert _retry_after_seconds(resp, 1) == 2.0
assert _retry_after_seconds(resp, 2) == 4.0
assert _retry_after_seconds(resp, 3) == 8.0
assert retry_after_seconds(resp, 1) == 2.0
assert retry_after_seconds(resp, 2) == 4.0
assert retry_after_seconds(resp, 3) == 8.0
def test_retry_after_seconds_caps():
assert _retry_after_seconds(_FakeResp(429), 10) == 30.0
assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0
assert retry_after_seconds(_FakeResp(429), 10) == 30.0
assert retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0
def _client_with_sequence(monkeypatch, responses):
+10 -9
View File
@@ -14,13 +14,14 @@ import pytest
import requests
from backend.app.services import patreon_downloader as pd_mod
from backend.app.services.patreon_client import MediaItem
from backend.app.services.patreon_downloader import (
from backend.app.services.native_ingest_common import (
MediaOutcome,
PatreonDownloader,
PostRecordOutcome,
_sanitize,
post_dir_name,
sanitize_segment,
)
from backend.app.services.patreon_client import MediaItem
from backend.app.services.patreon_downloader import PatreonDownloader
from backend.app.utils.sidecar import find_sidecar
# Minimal valid PNG so the file_validator passes on the happy path.
@@ -414,10 +415,10 @@ def test_validation_off_keeps_bytes(tmp_path):
def test_sanitize_helper():
assert _sanitize('a/b') == "a_b"
assert _sanitize('x<>:"|?*y') == "x_______y"
assert _sanitize("trail. ") == "trail"
assert _sanitize("") == "_"
assert sanitize_segment('a/b') == "a_b"
assert sanitize_segment('x<>:"|?*y') == "x_______y"
assert sanitize_segment("trail. ") == "trail"
assert sanitize_segment("") == "_"
def test_media_outcome_shape():
@@ -613,7 +614,7 @@ def test_media_sidecar_is_minimal_post_first(tmp_path):
post["attributes"]["content"] = "" # even an empty feed body isn't fetched here
dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x")
post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post)
post_dir = tmp_path / "artist-x" / "patreon" / post_dir_name(post)
sc = find_sidecar(post_dir / "01_a.png")
assert sc is not None
data = json.loads(sc.read_text())
+3 -5
View File
@@ -14,6 +14,7 @@ from pathlib import Path
import requests
from backend.app.services.gallery_dl import DownloadResult, ErrorType
from backend.app.services.native_ingest_common import post_dir_name
from backend.app.services.subscribestar_client import (
MediaItem,
SubscribeStarAPIError,
@@ -22,10 +23,7 @@ from backend.app.services.subscribestar_client import (
SubscribeStarDriftError,
_parse_ss_datetime,
)
from backend.app.services.subscribestar_downloader import (
SubscribeStarDownloader,
_post_dir_name,
)
from backend.app.services.subscribestar_downloader import SubscribeStarDownloader
from backend.app.services.subscribestar_ingester import (
SubscribeStarIngester,
_ledger_key,
@@ -210,7 +208,7 @@ def _downloader(tmp_path: Path, session=None, validate=True):
def test_post_dir_name_empty_title_matches_gallery_dl():
# SubscribeStar has no title → "<date>_<id>_" (matches gallery-dl's layout so
# existing downloads dedup on disk at cutover).
assert _post_dir_name(_post("111")) == "2026-05-01_111_"
assert post_dir_name(_post("111")) == "2026-05-01_111_"
def test_download_post_layout_and_outcomes(tmp_path):