bvandeusen
ebe6ab9741
DRY pass commit 2. The two adapters re-implemented the same auth→drift→429→404
→http→network mapping in _failure_result; only the exception classes + drift
phrasing differed (divergence-bug risk: a new error_type handled in one and not
the other).
- native_ingest_common gains NativeIngestError / NativeAuthError / NativeDriftError
(status_code + retry_after on the base). Patreon{API,Auth,Drift}Error and
SubscribeStar{API,Auth,Drift}Error now subclass them via multiple inheritance,
keeping their isinstance-distinct platform names.
- Ingester._failure_result (base) does the whole mapping via the shared
NativeAuthError/NativeDriftError taxonomy + status_code; a new platform gets it
free. New drift_label kwarg supplies the per-platform API_DRIFT phrasing
("Patreon API" / "SubscribeStar markup"), preserving the existing message
(test asserts "Patreon API changed").
- Both adapters drop their near-identical _failure_result overrides and their now
-unused DownloadResult/ErrorType/*Auth/*Drift imports.
Verified at every consumer (rule 93/§8b): test_patreon_ingester (auth/drift/429/
404/network) and test_subscribestar_native (_failure_result mapping) both exercise
the base method now. Remaining: ingest_core L1/L3 logging (3/3).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
110 lines
4.3 KiB
Python
110 lines
4.3 KiB
Python
"""Native SubscribeStar ingester — the SubscribeStar ADAPTER over the
|
|
platform-agnostic core (`ingest_core.Ingester`).
|
|
|
|
Thin counterpart to patreon_ingester: wires the SubscribeStar client/downloader/
|
|
ledger models/constraints/key into the core and supplies the SubscribeStar
|
|
failure mapping. The three modes (tick / backfill / recovery / recapture), the
|
|
seen + dead-letter ledgers, cursor checkpointing, and the post-first capture all
|
|
live in the core — identical to Patreon. `download_service.download_source`
|
|
drives `SubscribeStarIngester.run` exactly as it drives the Patreon one.
|
|
|
|
`campaign_id` is the creator URL (the client derives host + slug from it), so no
|
|
campaign-id resolver is needed. FC runs on a plain-HTTP homelab; nothing here
|
|
uses a secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
|
|
from ..models import SubscribeStarFailedMedia, SubscribeStarSeenMedia
|
|
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
|
from .subscribestar_client import MediaItem, SubscribeStarAPIError, SubscribeStarClient
|
|
from .subscribestar_downloader import SubscribeStarDownloader
|
|
|
|
__all__ = [
|
|
"DEAD_LETTER_THRESHOLD",
|
|
"SubscribeStarIngester",
|
|
"_ledger_key",
|
|
"verify_subscribestar_credential",
|
|
]
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_LEDGER_KEY_MAX = 128
|
|
|
|
|
|
def _ledger_key(media: MediaItem) -> str:
|
|
"""Stable per-media identity for the cross-run seen-ledger. SubscribeStar's
|
|
full-res URL is an opaque `/post_uploads?payload=...` (no content hash), so
|
|
`media.filehash` is normally None and the stable proxy is the gallery item id
|
|
scoped to its post: `<post_id>:<media_id>`. Bounded to the column width."""
|
|
if media.filehash:
|
|
return media.filehash
|
|
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
|
|
|
|
|
class SubscribeStarIngester(Ingester):
|
|
"""Walk a SubscribeStar creator's posts, download unseen media, return a
|
|
`DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
|
|
`downloader` are injectable seams so unit tests run without network."""
|
|
|
|
def __init__(
|
|
self,
|
|
images_root: Path,
|
|
cookies_path: str | None,
|
|
session_factory: Callable[[], object],
|
|
*,
|
|
validate: bool = True,
|
|
rate_limit: float = 0.0,
|
|
request_sleep: float = 0.0,
|
|
client: SubscribeStarClient | None = None,
|
|
downloader: SubscribeStarDownloader | None = None,
|
|
):
|
|
self.images_root = Path(images_root)
|
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
|
resolved_client = (
|
|
client
|
|
if client is not None
|
|
else SubscribeStarClient(cookies_path, request_sleep=request_sleep)
|
|
)
|
|
resolved_downloader = (
|
|
downloader
|
|
if downloader is not None
|
|
else SubscribeStarDownloader(
|
|
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
|
)
|
|
)
|
|
super().__init__(
|
|
client=resolved_client,
|
|
downloader=resolved_downloader,
|
|
session_factory=session_factory,
|
|
seen_model=SubscribeStarSeenMedia,
|
|
failed_model=SubscribeStarFailedMedia,
|
|
seen_constraint="uq_subscribestar_seen_media_source_id",
|
|
failed_constraint="uq_subscribestar_failed_media_source_id",
|
|
ledger_key=_ledger_key,
|
|
platform="subscribestar",
|
|
error_base=SubscribeStarAPIError,
|
|
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
|
# the auth/drift/HTTP→error_type mapping (shared across platforms).
|
|
drift_label="SubscribeStar markup",
|
|
)
|
|
|
|
|
|
async def verify_subscribestar_credential(
|
|
url: str,
|
|
cookies_path: str | None,
|
|
overrides: dict | None,
|
|
) -> tuple[bool | None, str]:
|
|
"""Native SubscribeStar credential probe — fetches ONE feed page via
|
|
SubscribeStarClient.verify_auth. `campaign_id` is just the creator URL (no
|
|
resolver). Returns the uniform `(ok, message)` contract so
|
|
download_backends.verify_credential treats it like the gallery-dl probe."""
|
|
client = SubscribeStarClient(cookies_path)
|
|
loop = asyncio.get_running_loop()
|
|
return await loop.run_in_executor(None, client.verify_auth, url)
|