refactor(native-ingest): extract native_ingest_common + BaseNativeDownloader (#899 DRY 1/3)
DRY pass commit 1 (process #594). Consolidate the helpers + download plumbing the Patreon and SubscribeStar adapters had duplicated (SubscribeStar was importing patreon privates — wrong owner). New backend/app/services/ native_ingest_common.py is the neutral home for: - make_session (was _load_session ×2), retry_after_seconds + 429 constants, sanitize_segment, basename_from_url, post_dir_name, MediaOutcome / PostRecordOutcome. - BaseNativeDownloader: the shared streaming GET (transient-retry + Range-resume) and validation/quarantine. Patreon + SubscribeStar downloaders now subclass it; each keeps only what differs (Patreon's Mux/yt-dlp video branch + detail-fetch enrichment; SubscribeStar nothing extra). Behavior preserved exactly; the divergence-bug risk (a fix to one _fetch_to_file not reaching the other) is gone. - Folds in #899 L2: a quarantine now log.warning's path+reason (was counted only). post_dir_name merges both date handlers (accepts trailing-Z and pre-parsed ISO). Tests repointed to the single source at every consumer (rule 93 / §8b parity): patreon_client/downloader, subscribestar_native. Exception-trio consolidation + base _failure_result (2/3) and the remaining ingest_core logging (3/3) follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -14,13 +14,14 @@ import pytest
|
||||
import requests
|
||||
|
||||
from backend.app.services import patreon_downloader as pd_mod
|
||||
from backend.app.services.patreon_client import MediaItem
|
||||
from backend.app.services.patreon_downloader import (
|
||||
from backend.app.services.native_ingest_common import (
|
||||
MediaOutcome,
|
||||
PatreonDownloader,
|
||||
PostRecordOutcome,
|
||||
_sanitize,
|
||||
post_dir_name,
|
||||
sanitize_segment,
|
||||
)
|
||||
from backend.app.services.patreon_client import MediaItem
|
||||
from backend.app.services.patreon_downloader import PatreonDownloader
|
||||
from backend.app.utils.sidecar import find_sidecar
|
||||
|
||||
# Minimal valid PNG so the file_validator passes on the happy path.
|
||||
@@ -414,10 +415,10 @@ def test_validation_off_keeps_bytes(tmp_path):
|
||||
|
||||
|
||||
def test_sanitize_helper():
|
||||
assert _sanitize('a/b') == "a_b"
|
||||
assert _sanitize('x<>:"|?*y') == "x_______y"
|
||||
assert _sanitize("trail. ") == "trail"
|
||||
assert _sanitize("") == "_"
|
||||
assert sanitize_segment('a/b') == "a_b"
|
||||
assert sanitize_segment('x<>:"|?*y') == "x_______y"
|
||||
assert sanitize_segment("trail. ") == "trail"
|
||||
assert sanitize_segment("") == "_"
|
||||
|
||||
|
||||
def test_media_outcome_shape():
|
||||
@@ -613,7 +614,7 @@ def test_media_sidecar_is_minimal_post_first(tmp_path):
|
||||
post["attributes"]["content"] = "" # even an empty feed body isn't fetched here
|
||||
dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x")
|
||||
|
||||
post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post)
|
||||
post_dir = tmp_path / "artist-x" / "patreon" / post_dir_name(post)
|
||||
sc = find_sidecar(post_dir / "01_a.png")
|
||||
assert sc is not None
|
||||
data = json.loads(sc.read_text())
|
||||
|
||||
Reference in New Issue
Block a user