diff --git a/alembic/versions/0037_patreon_seen_media.py b/alembic/versions/0037_patreon_seen_media.py new file mode 100644 index 0000000..255484e --- /dev/null +++ b/alembic/versions/0037_patreon_seen_media.py @@ -0,0 +1,53 @@ +"""patreon_seen_media: per-source ledger of already-ingested Patreon media + +Revision ID: 0037 +Revises: 0036 +Create Date: 2026-06-05 + +Native Patreon ingester (build step 2a). Replaces gallery-dl's +archive.sqlite3 with our own queryable table. The downloader upserts one +row per (source, media) so routine walks skip media we've already +processed; a future "recovery" mode bypasses the ledger to re-walk. + +`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form +``video::`` — hence String(128). The unique +constraint on (source_id, filehash) is the dedup upsert key. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0037" +down_revision: Union[str, None] = "0036" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_seen_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_seen_media") diff --git a/alembic/versions/0038_patreon_failed_media.py b/alembic/versions/0038_patreon_failed_media.py new file mode 100644 index 0000000..e907ae1 --- /dev/null +++ b/alembic/versions/0038_patreon_failed_media.py @@ -0,0 +1,58 @@ +"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media + +Revision ID: 0038 +Revises: 0037 +Create Date: 2026-06-06 + +Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN, +deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here +with an attempt counter; once it crosses the dead-letter threshold the ingester +skips it on routine walks (recovery still re-attempts). A clean download clears +the row. UNIQUE (source_id, filehash) is the upsert key (same media key the +seen-ledger uses). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0038" +down_revision: Union[str, None] = "0037" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_failed_media") diff --git a/backend/app/api/credentials.py b/backend/app/api/credentials.py index f719b51..a7136dc 100644 --- a/backend/app/api/credentials.py +++ b/backend/app/api/credentials.py @@ -121,13 +121,15 @@ async def delete_credential(platform: str): @credentials_bp.route("//verify", methods=["POST"]) async def verify_credential(platform: str): - """Test the stored credential by running gallery-dl --simulate - against one of the platform's enabled sources. On success stamps - last_verified. Returns {valid: bool|null, reason, last_verified?}. - valid=null means "couldn't test" (no credential, or no enabled - source to point at).""" + """Test the stored credential against one of the platform's enabled sources, + WITHOUT downloading. Routes through the platform's backend + (download_backends.verify_credential) — native ingester for Patreon, an + authenticated API page; gallery-dl --simulate for the rest. On success + stamps last_verified. Returns {valid: bool|null, reason, last_verified?}; + valid=null means "couldn't test" (no credential, no enabled source, or an + inconclusive network/drift result).""" from ..models import Artist, Source - from ..services.gallery_dl import GalleryDLService, SourceConfig + from ..services.download_backends import verify_source_credential async with get_session() as session: if not await _ext_key_ok(session): @@ -154,14 +156,14 @@ async def verify_credential(platform: str): cookies_path = await svc.get_cookies_path(platform) auth_token = await svc.get_token(platform) - gdl = GalleryDLService(images_root=Path("/images")) - ok, message = await gdl.verify( + ok, message = await verify_source_credential( + platform=platform, url=source.url, artist_slug=artist.slug, - platform=platform, - source_config=SourceConfig.from_dict(source.config_overrides or {}), + config_overrides=source.config_overrides or {}, cookies_path=str(cookies_path) if cookies_path else None, auth_token=auth_token, + images_root=Path("/images"), ) last_verified = None diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index d33e317..d5871e1 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -122,28 +122,119 @@ async def delete_source(source_id: int): @sources_bp.route("//backfill", methods=["POST"]) async def set_backfill(source_id: int): - """Plan #693: start or stop a run-until-done backfill. Body: - `{"action": "start" | "stop"}` (default "start"). 'start' walks the full - post history in time-boxed chunks until it reaches the bottom (then the - source shows 'complete'); 'stop' cancels back to tick mode. Returns the - updated source dict (incl. backfill_state / backfill_chunks).""" + """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery. + Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start' + walks the full post history in time-boxed chunks until it reaches the bottom + (then the source shows 'complete'); 'recover' is the same walk but bypasses + the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the + current pHash threshold; 'stop' cancels either back to tick mode. Returns the + updated source dict (incl. backfill_state / backfill_chunks / + backfill_bypass_seen).""" + from pathlib import Path + + from ..services.credential_service import CredentialService + from ..services.download_backends import ( + uses_native_ingester, + verify_source_credential, + ) + from .credentials import _get_crypto + payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") - if action not in ("start", "stop"): - return _bad("invalid_action", detail="action must be 'start' or 'stop'") + if action not in ("start", "stop", "recover"): + return _bad( + "invalid_action", + detail="action must be 'start', 'stop', or 'recover'", + ) + + # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester + # platform (where verify is one cheap API page), refuse if the credential is + # DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed + # on valid OR inconclusive (a network blip shouldn't block). Gated to native + # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for + # an arm action. The credential read happens in a session that's CLOSED + # before the verify network call (don't hold a DB conn across the request). + if action in ("start", "recover"): + async with get_session() as session: + rec = await SourceService(session).get(source_id) + if rec is None: + return _bad("not_found", status=404) + native = uses_native_ingester(rec.platform) + if native: + cred = CredentialService(session, _get_crypto()) + cookies_path = await cred.get_cookies_path(rec.platform) + auth_token = await cred.get_token(rec.platform) + if native: + ok, message = await verify_source_credential( + platform=rec.platform, + url=rec.url, + artist_slug=rec.artist_slug, + config_overrides=rec.config_overrides or {}, + cookies_path=str(cookies_path) if cookies_path else None, + auth_token=auth_token, + images_root=Path("/images"), + ) + if ok is False: + return _bad("credential_rejected", detail=message, status=409) + async with get_session() as session: try: svc = SourceService(session) - record = ( - await svc.start_backfill(source_id) - if action == "start" - else await svc.stop_backfill(source_id) - ) + if action == "start": + record = await svc.start_backfill(source_id) + elif action == "recover": + record = await svc.start_recovery(source_id) + else: + record = await svc.stop_backfill(source_id) except LookupError: return _bad("not_found", status=404) return jsonify(record.to_dict()) +@sources_bp.route("//preview", methods=["POST"]) +async def preview_source_endpoint(source_id: int): + """Plan #708 B4: dry-run — count what a backfill WOULD download for a native + platform (Patreon today), without downloading. Walks the first few feed pages + and counts media not already in the seen/dead ledgers. Returns + {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason + (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no + cheap dry-run — their verify is a slow --simulate).""" + from pathlib import Path + + from ..services.credential_service import CredentialService + from ..services.download_backends import preview_source, uses_native_ingester + from ..tasks._sync_engine import sync_session_factory + from .credentials import _get_crypto + + async with get_session() as session: + rec = await SourceService(session).get(source_id) + if rec is None: + return _bad("not_found", status=404) + if not uses_native_ingester(rec.platform): + return _bad( + "unsupported", + detail="Preview is only available for native-ingester platforms.", + status=400, + ) + cred = CredentialService(session, _get_crypto()) + cookies_path = await cred.get_cookies_path(rec.platform) + + # The walk + ledger reads are sync (run off the request loop); the process + # sync engine is the same one the download task uses. + result = await preview_source( + platform=rec.platform, + url=rec.url, + source_id=source_id, + config_overrides=rec.config_overrides or {}, + cookies_path=str(cookies_path) if cookies_path else None, + images_root=Path("/images"), + sync_session_factory=sync_session_factory(), + ) + if "error" in result: + return _bad("preview_failed", detail=result["error"], status=409) + return jsonify(result) + + @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 137311c..b8854b1 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -14,6 +14,8 @@ from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun from .ml_settings import MLSettings +from .patreon_failed_media import PatreonFailedMedia +from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment from .series_page import SeriesPage @@ -33,6 +35,8 @@ __all__ = [ "BackupRun", "Source", "Credential", + "PatreonFailedMedia", + "PatreonSeenMedia", "Post", "PostAttachment", "SeriesPage", diff --git a/backend/app/models/patreon_failed_media.py b/backend/app/models/patreon_failed_media.py new file mode 100644 index 0000000..79976ef --- /dev/null +++ b/backend/app/models/patreon_failed_media.py @@ -0,0 +1,45 @@ +"""PatreonFailedMedia — per-source dead-letter ledger of Patreon media that +keeps failing to download/validate. + +Plan #705 (#7). A media that fails every walk (404'd CDN URL, deleted post, +geo-blocked Mux stream, persistently-corrupt bytes) would otherwise re-error +forever and re-burn backfill chunks. After ``attempts`` reaches the dead-letter +threshold the ingester skips it on routine tick/backfill walks (recovery still +re-attempts it — the operator's "try everything again"). A later clean download +clears the row (the media recovered). + +`filehash` is the same per-media key the seen-ledger uses (32-hex CDN MD5, or a +``video:`` / ``post:filename`` synthesized key) — hence String(128). UNIQUE +(source_id, filehash) is the upsert key. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class PatreonFailedMedia(Base): + __tablename__ = "patreon_failed_media" + __table_args__ = ( + UniqueConstraint( + "source_id", "filehash", name="uq_patreon_failed_media_source_id" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + first_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + last_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/patreon_seen_media.py b/backend/app/models/patreon_seen_media.py new file mode 100644 index 0000000..9ba92f2 --- /dev/null +++ b/backend/app/models/patreon_seen_media.py @@ -0,0 +1,38 @@ +"""PatreonSeenMedia — per-source ledger of Patreon media already +downloaded+processed. + +Replaces gallery-dl's archive.sqlite3 with our own queryable table so +routine walks can skip media we've already ingested (and a future +"recovery" mode can deliberately bypass the ledger to re-walk). + +`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos — +which have no stable content hash at discovery time — use a sentinel of +the form ``video::``, hence String(128) rather than 32. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class PatreonSeenMedia(Base): + __tablename__ = "patreon_seen_media" + __table_args__ = ( + # Dedup key the downloader upserts against: one ledger row per + # (source, media). A second sighting of the same media is a no-op. + UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + post_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py new file mode 100644 index 0000000..02095f8 --- /dev/null +++ b/backend/app/services/download_backends.py @@ -0,0 +1,229 @@ +"""Platform → download-backend dispatch (one place that knows which platforms +are served by the native FC ingester vs. the gallery-dl subprocess). + +gallery-dl wasn't built to be driven by an automated scheduler — no native +checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock. +The native ingester (services/patreon_ingester.py, plan #697) replaces it for +Patreon and is the path we grow as more platforms migrate. To keep that +migration DRY, every caller that has to behave differently per backend — +download routing, the credential-verify probe, cursor handling — asks THIS +module instead of testing ``platform == "patreon"`` inline. When a platform gets +a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the +download path and verify switch over together. + +The backend surfaces share a UNIFORM signature so a caller invokes the same +function regardless of platform: + - verify_credential(...) → (ok: bool|None, message: str) + - (download stays in download_service for now; uses_native_ingester() is the + shared predicate it routes on, so the decision lives here too.) +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from .gallery_dl import DownloadResult, ErrorType +from .patreon_ingester import PatreonIngester +from .patreon_resolver import resolve_campaign_id_for_source + +# Platforms whose download + verify go through the native ingester rather than +# gallery-dl. gallery-dl still serves every other platform (subscribestar, +# hentaifoundry, discord, pixiv, deviantart) unchanged. +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"}) + + +def uses_native_ingester(platform: str) -> bool: + """True when `platform` is served by the native ingester (not gallery-dl). + The single predicate the download path and verify both route on.""" + return platform in NATIVE_INGESTER_PLATFORMS + + +async def run_download( + *, + ctx: dict, + source_config, + skip_value: bool | str, + mode: str | None, + gdl, + sync_session_factory, +) -> tuple[DownloadResult, str | None]: + """Uniform download across backends — the download counterpart to + `verify_source_credential`, so this module is the ONE place that knows how + each platform both downloads AND verifies (the seam that makes adding a + platform a bounded job). + + Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is + non-None only when a native vanity lookup ran this call (so phase 3 caches + it). Native platforms route through their ingester in `mode` + (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller + (download_service) prepares `source_config`/`skip_value`/`mode` from the + backfill state machine and owns phase 3. + """ + platform = ctx["platform"] + if uses_native_ingester(platform): + return await _run_native_ingester( + ctx, source_config, mode, gdl, sync_session_factory + ) + result = await gdl.download( + url=ctx["url"], + artist_slug=ctx["artist_slug"], + platform=platform, + source_config=source_config, + cookies_path=ctx["cookies_path"], + auth_token=ctx["auth_token"], + skip_value=skip_value, + ) + return result, None + + +async def _run_native_ingester( + ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, +) -> tuple[DownloadResult, str | None]: + """Patreon (today the only native platform): resolve the campaign id, then run + the native ingester in a worker thread (it is sync requests/subprocess). + + `resolved_campaign_id` is non-None only when we had to look it up from the + vanity URL this run, so phase 3 caches it the way the old gallery-dl retry + did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent + empty success. + """ + overrides = ctx["config_overrides"] or {} + campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( + ctx["url"], ctx["cookies_path"], overrides + ) + if not campaign_id: + return ( + DownloadResult( + success=False, + url=ctx["url"], + artist_slug=ctx["artist_slug"], + platform="patreon", + error_type=ErrorType.NOT_FOUND, + error_message=( + "Could not resolve Patreon campaign id from the source URL " + "(vanity lookup failed — cookies expired or creator moved?)" + ), + ), + None, + ) + + # Honor the operator's existing rate-limit knobs on the native path (plan + # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`, + # here on the gdl service) paces media downloads; page fetches use the + # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same + # API-pacing default gallery-dl applied as its `sleep-request`. + rate_limit = gdl._rate_limit + request_sleep = ( + source_config.sleep_request + if source_config.sleep_request is not None + else max(0.5, rate_limit / 4) + ) + ingester = PatreonIngester( + images_root=gdl.images_root, + cookies_path=ctx["cookies_path"], + session_factory=sync_session_factory, + validate=gdl._validate_files, + rate_limit=rate_limit, + request_sleep=request_sleep, + ) + loop = asyncio.get_running_loop() + dl_result = await loop.run_in_executor( + None, + lambda: ingester.run( + source_id=ctx["source_id"], + campaign_id=campaign_id, + artist_slug=ctx["artist_slug"], + url=ctx["url"], + mode=mode, + resume_cursor=source_config.resume_cursor, + time_budget_seconds=source_config.timeout, + posts_base=int(overrides.get("_backfill_posts", 0)), + ), + ) + return dl_result, resolved_campaign_id + + +async def preview_source( + *, + platform: str, + url: str, + source_id: int, + config_overrides: dict | None, + cookies_path: str | None, + images_root: Path, + sync_session_factory, + page_limit: int = 3, +) -> dict: + """Dry-run preview for a native platform (plan #708 B4): resolve the campaign + id, then walk a few pages counting media not already seen/dead — no download. + + Returns the preview dict (total_new / posts_scanned / pages_scanned / + has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. + Native-only — the caller gates on `uses_native_ingester`. + """ + import asyncio + + from .patreon_client import PatreonAPIError + + campaign_id, _ = await resolve_campaign_id_for_source( + url, cookies_path, config_overrides or {} + ) + if not campaign_id: + return { + "error": ( + "Couldn't resolve the campaign id from the source URL " + "(cookies expired, or the creator moved/renamed?)." + ) + } + ingester = PatreonIngester( + images_root=images_root, + cookies_path=cookies_path, + session_factory=sync_session_factory, + ) + loop = asyncio.get_running_loop() + try: + result = await loop.run_in_executor( + None, + lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), + ) + except PatreonAPIError as exc: + return {"error": f"Couldn't preview: {exc}"} + return result + + +async def verify_source_credential( + *, + platform: str, + url: str, + artist_slug: str, + config_overrides: dict | None, + cookies_path: str | None, + auth_token: str | None, + images_root: Path, +) -> tuple[bool | None, str]: + """Uniform credential probe across backends. Returns `(ok, message)`: + True = authenticated, False = rejected, None = inconclusive (drift / + network / nothing to test). Callers don't branch on platform — they call + this and render the result. + """ + if uses_native_ingester(platform): + # Native ingester platforms verify via their own lightweight auth probe + # (resolve campaign id + one authenticated API page). Patreon today. + from .patreon_ingester import verify_patreon_credential + + return await verify_patreon_credential(url, cookies_path, config_overrides) + + # gallery-dl platforms: --simulate one item; the extractor errors before it + # can list if auth is bad. + from .gallery_dl import GalleryDLService, SourceConfig + + gdl = GalleryDLService(images_root=images_root) + return await gdl.verify( + url=url, + artist_slug=artist_slug, + platform=platform, + source_config=SourceConfig.from_dict(config_overrides or {}), + cookies_path=cookies_path, + auth_token=auth_token, + ) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 8804483..66e960b 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio import logging -import re from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -25,45 +24,25 @@ from sqlalchemy.orm import joinedload from ..models import Artist, DownloadEvent, Source from .credential_service import CredentialService +from .download_backends import run_download, uses_native_ingester from .gallery_dl import ( BACKFILL_CHUNK_SECONDS, BACKFILL_SKIP_VALUE, TICK_SKIP_VALUE, + DownloadResult, ErrorType, GalleryDLService, SourceConfig, - parse_last_cursor, + extract_errors_warnings, + truncate_log, ) from .importer import Importer -from .patreon_resolver import resolve_campaign_id from .platforms import auth_type_for from .scheduler_service import set_platform_cooldown log = logging.getLogger(__name__) -_PATREON_VANITY_RE = re.compile( - r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", - re.IGNORECASE, -) -_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id" - - -def _extract_patreon_vanity(url: str) -> str | None: - m = _PATREON_VANITY_RE.match(url) - return m.group(1) if m else None - - -def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool: - return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower() - - -def _effective_url(platform: str, source_url: str, overrides: dict) -> str: - if platform == "patreon" and overrides.get("patreon_campaign_id"): - return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}" - return source_url - - class DownloadService: """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`. @@ -78,12 +57,18 @@ class DownloadService: gdl: GalleryDLService, importer: Importer, cred_service: CredentialService, + sync_session_factory=None, ): self.async_session = async_session self.sync_session = sync_session self.gdl = gdl self.importer = importer self.cred_service = cred_service + # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED + # sessions from for its seen-ledger reads/writes (never one held across + # the multi-minute walk — see PatreonIngester). Only the patreon branch + # of phase 2 uses it; gallery-dl sources leave it None. + self.sync_session_factory = sync_session_factory async def download_source(self, source_id: int) -> int: """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is.""" @@ -118,56 +103,41 @@ class DownloadService: # Operator drives this via POST /api/sources/{id}/backfill {action}. overrides = ctx["config_overrides"] or {} in_backfill = overrides.get("_backfill_state") == "running" + # Recovery (plan #697) reuses the entire #693 backfill state machine — + # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and + # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted + # near-dups get re-fetched and re-evaluated under the CURRENT pHash + # threshold (tier-2 disk still spares files we kept). The + # `_backfill_bypass_seen` flag rides alongside the running backfill state; + # download mode is "recovery" when both are set. + bypass_seen = bool(overrides.get("_backfill_bypass_seen")) if in_backfill: skip_value: bool | str = BACKFILL_SKIP_VALUE source_config.timeout = BACKFILL_CHUNK_SECONDS pending_cursor = overrides.get("_backfill_cursor") - if ctx["platform"] == "patreon" and pending_cursor: + if uses_native_ingester(ctx["platform"]) and pending_cursor: source_config.resume_cursor = pending_cursor else: skip_value = TICK_SKIP_VALUE - effective_url = _effective_url( - ctx["platform"], ctx["url"], ctx["config_overrides"] or {} + # Phase 2 dispatch is uniform across backends (download_backends. + # run_download — the download counterpart to verify_source_credential): + # native platforms run their ingester in `mode` (zero per-file HEADs, + # native cursor/resume, loud drift detection); gallery-dl platforms run + # the subprocess. Either returns a DownloadResult-shaped object so phase 3 + # is untouched. `mode` is None for gallery-dl (run_download ignores it). + mode: str | None = None + if uses_native_ingester(ctx["platform"]): + if in_backfill and bypass_seen: + mode = "recovery" + elif in_backfill: + mode = "backfill" + else: + mode = "tick" + dl_result, resolved_campaign_id = await self._run_download( + ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode, ) - dl_result = await self.gdl.download( - url=effective_url, - artist_slug=ctx["artist_slug"], - platform=ctx["platform"], - source_config=source_config, - cookies_path=ctx["cookies_path"], - auth_token=ctx["auth_token"], - skip_value=skip_value, - ) - - resolved_campaign_id: str | None = None - if ( - ctx["platform"] == "patreon" - and not dl_result.success - and "patreon_campaign_id" not in (ctx["config_overrides"] or {}) - and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr) - ): - vanity = _extract_patreon_vanity(ctx["url"]) - if vanity: - log.info( - "Attempting campaign-ID resolution for %s (%s)", - ctx["artist_slug"], vanity, - ) - resolved_campaign_id = await resolve_campaign_id( - vanity, ctx["cookies_path"] - ) - if resolved_campaign_id: - dl_result = await self.gdl.download( - url=f"https://www.patreon.com/id:{resolved_campaign_id}", - artist_slug=ctx["artist_slug"], - platform=ctx["platform"], - source_config=source_config, - cookies_path=ctx["cookies_path"], - auth_token=ctx["auth_token"], - skip_value=skip_value, - ) - # A backfill chunk that hit its time-box but made forward progress is # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as # "ok/progress" (PARTIAL maps to status "ok"), not a red error, and the @@ -175,7 +145,7 @@ class DownloadService: # timed out with NO progress stays TIMEOUT and feeds phase 3's # stall-guard. Leave RATE_LIMITED alone so the platform-cooldown fires. if in_backfill and dl_result.error_type == ErrorType.TIMEOUT: - new_cursor = parse_last_cursor(dl_result.stdout, dl_result.stderr) + new_cursor = dl_result.cursor # plan #704: structured, not scraped advanced = bool( (new_cursor and new_cursor != overrides.get("_backfill_cursor")) or dl_result.files_downloaded > 0 @@ -190,6 +160,18 @@ class DownloadService: ctx["event_id"], ctx, dl_result, resolved_campaign_id, ) + async def _run_download( + self, *, ctx: dict, source_config, skip_value, mode: str | None, + ) -> tuple[DownloadResult, str | None]: + """Phase-2 dispatch → `download_backends.run_download`, passing this + service's gdl + sync sessionmaker. Kept as a thin instance method so tests + can stub the whole phase-2 dispatch on the service, and so the per-backend + construction lives in download_backends (the backend registry).""" + return await run_download( + ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode, + gdl=self.gdl, sync_session_factory=self.sync_session_factory, + ) + async def _phase1_setup(self, source_id: int) -> dict[str, Any]: source = (await self.async_session.execute( select(Source).options(joinedload(Source.artist)).where(Source.id == source_id) @@ -386,11 +368,16 @@ class DownloadService: select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() - run_stats = self.gdl._compute_run_stats( - dl_result.return_code, dl_result.stdout, dl_result.stderr - ) + # plan #704: the native ingester returns structured run_stats; only the + # gallery-dl path needs the regex-over-stdout reconstruction. + if dl_result.run_stats is not None: + run_stats = dict(dl_result.run_stats) + else: + run_stats = self.gdl._compute_run_stats( + dl_result.return_code, dl_result.stdout, dl_result.stderr + ) run_stats["quarantined_count"] = dl_result.files_quarantined - stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) + stderr_summary = extract_errors_warnings(dl_result.stderr) # Plan #544: PARTIAL means the run downloaded ≥1 file but the # subprocess didn't finish in budget (typically wall-clock timeout @@ -410,8 +397,8 @@ class DownloadService: ev.metadata_ = { "run_stats": run_stats, "error_type": dl_result.error_type.value if dl_result.error_type else None, - "stdout": self.gdl._truncate_log(dl_result.stdout) or None, - "stderr": self.gdl._truncate_log(dl_result.stderr) or None, + "stdout": truncate_log(dl_result.stdout) or None, + "stderr": truncate_log(dl_result.stderr) or None, "stderr_errors_warnings": stderr_summary or None, "duration_seconds": dl_result.duration_seconds, "quarantined_paths": dl_result.quarantined_paths or None, @@ -420,6 +407,7 @@ class DownloadService: await self._update_source_health( source_id=ctx["source_id"], status=status, error_message=ev.error, error_type=dl_result.error_type.value if dl_result.error_type else None, + retry_after_seconds=getattr(dl_result, "retry_after_seconds", None), ) await self._apply_backfill_lifecycle(ctx, dl_result) await self.async_session.commit() @@ -458,6 +446,12 @@ class DownloadService: new_overrides = dict(src.config_overrides or {}) chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1 new_overrides["_backfill_chunks"] = chunks + # plan #704 (#5): _backfill_posts (the live progress badge) is OWNED by the + # ingester now — it writes a monotonic absolute mid-walk at each page + # boundary (ingest_core._checkpoint_posts), so the badge climbs DURING a + # chunk instead of jumping once per chunk here, and the re-walked resume + # page is no longer double-counted. new_overrides (read fresh above) + # carries the ingester's committed value forward untouched. completed = ( dl_result.success @@ -468,16 +462,19 @@ class DownloadService: new_overrides["_backfill_state"] = "complete" new_overrides.pop("_backfill_cursor", None) new_overrides.pop("_backfill_cursor_stalls", None) + # plan #697: a recovery walk shares this lifecycle; clear its bypass + # flag on completion so the next routine tick honors the seen-ledger. + new_overrides.pop("_backfill_bypass_seen", None) src.config_overrides = new_overrides src.backfill_runs_remaining = 0 return - # Did not finish. Patreon checkpoints + resumes via cursor; other - # platforms have no resumable cursor (every chunk re-walks from the - # top), so they advance only by the download archive growing. + # Did not finish. The native ingester checkpoints + resumes via cursor + # (carried structurally on the result, plan #704); gallery-dl platforms + # have no resumable cursor (every chunk re-walks from the top), so they + # advance only by the download archive growing. new_cursor = ( - parse_last_cursor(dl_result.stdout, dl_result.stderr) - if ctx["platform"] == "patreon" else None + dl_result.cursor if uses_native_ingester(ctx["platform"]) else None ) advanced = bool( (new_cursor and new_cursor != old_cursor) @@ -505,7 +502,7 @@ class DownloadService: async def _update_source_health( self, *, source_id: int, status: str, error_message: str | None, - error_type: str | None = None, + error_type: str | None = None, retry_after_seconds: float | None = None, ) -> None: """FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}. @@ -537,7 +534,17 @@ class DownloadService: # by error class without opening Logs per row. source.error_type = error_type if error_type == "rate_limited": - await set_platform_cooldown(self.async_session, source.platform) + # plan #708 B1: honor the server's Retry-After when the native + # client surfaced one (clamped to a sane [60, 3600] window so a + # tiny hint can't leave the platform effectively un-cooled and a + # huge one can't strand it for hours); else the flat default. + if retry_after_seconds is not None: + seconds = int(min(max(retry_after_seconds, 60), 3600)) + await set_platform_cooldown( + self.async_session, source.platform, seconds=seconds, + ) + else: + await set_platform_cooldown(self.async_session, source.platform) elif status == "skipped": source.last_error = None source.last_checked_at = now diff --git a/backend/app/services/file_validator.py b/backend/app/services/file_validator.py index ca6e40f..e007422 100644 --- a/backend/app/services/file_validator.py +++ b/backend/app/services/file_validator.py @@ -11,9 +11,15 @@ expected to write. from __future__ import annotations +import json +import logging +import shutil from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path +log = logging.getLogger(__name__) + JPEG_HEAD = b"\xff\xd8\xff" JPEG_TAIL = b"\xff\xd9" @@ -108,3 +114,52 @@ def validate_file(path: Path) -> ValidationResult: return ValidationResult(ok=True, format="webp", size=size) return ValidationResult(ok=True, format=None, size=size) + + +def quarantine_file( + images_root: Path, + path: Path, + artist_slug: str, + platform: str, + *, + url: str | None, + result: ValidationResult, +) -> Path | None: + """Move a validation-failed file to `_quarantine//` and write + a `.quarantine.json` provenance sidecar next to it. + + Returns the destination path, or None if the move itself failed (file left + in place — the caller decides what to report). The caller has already run + `validate_file` and seen `result.ok is False`. ONE implementation for both + download backends — gallery-dl's batch post-process and the native ingester's + per-media path — so the quarantine layout + provenance sidecar can't drift. + """ + quarantine_root = images_root / "_quarantine" / artist_slug / platform + try: + quarantine_root.mkdir(parents=True, exist_ok=True) + dest = quarantine_root / path.name + counter = 1 + while dest.exists(): + dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" + counter += 1 + shutil.move(str(path), str(dest)) + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + sidecar.write_text( + json.dumps( + { + "original_path": str(path), + "source_url": url, + "artist_slug": artist_slug, + "platform": platform, + "format": result.format, + "reason": result.reason, + "size": result.size, + "quarantined_at": datetime.now(UTC).isoformat(), + }, + indent=2, + ) + ) + except OSError as exc: + log.error("Failed to quarantine %s: %s. File left in place.", path, exc) + return None + return dest diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 9d59f3c..3add0e1 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio import json import logging -import re import subprocess import sys import tempfile @@ -23,7 +22,7 @@ from datetime import UTC, datetime from enum import StrEnum from pathlib import Path -from .file_validator import is_validatable, validate_file +from .file_validator import is_validatable, quarantine_file, validate_file log = logging.getLogger(__name__) @@ -40,6 +39,12 @@ class ErrorType(StrEnum): HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" VALIDATION_FAILED = "validation_failed" + # Native Patreon ingester only (plan #697): a response parsed as JSON but + # didn't match the JSON:API shape the ingester depends on (missing `data`, + # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix + # is updating the ingester's field-set/parser, not rotating credentials. The + # contract test guards against silently shipping this. + API_DRIFT = "api_drift" # Run made real progress (downloaded ≥1 file) but did not finish in the # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status # mapping classifies this as "ok" because the next tick continues. @@ -101,12 +106,14 @@ class SourceConfig: `resume_cursor` is RUNTIME state, not persisted operator config: the cursor-paged backfill checkpoint (see parse_last_cursor + the - download_service backfill lifecycle). When set on a patreon backfill - run, gallery-dl resumes its newest→oldest walk from that pagination - cursor instead of restarting from the top. It is threaded by - download_service from config_overrides["_backfill_cursor"]; SourceConfig - deliberately does NOT read it in from_dict (config_overrides also holds - operator config, and resume_cursor must only apply in backfill mode). + download_service backfill lifecycle). It is consumed only by the native + Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path + was removed at the #697 cutover): on a backfill/recovery run the ingester + resumes its newest→oldest walk from that pagination cursor instead of + restarting from the top. download_service threads it from + config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read + it in from_dict (config_overrides also holds operator config, and + resume_cursor must only apply in backfill/recovery mode). """ content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None @@ -148,6 +155,51 @@ class DownloadResult: duration_seconds: float = 0.0 started_at: str | None = None completed_at: str | None = None + # Plan #704 — structured fields the NATIVE ingester populates directly (None + # on the gallery-dl path, which keeps the regex-over-stdout route). When set, + # phase 3 reads these instead of scraping the text it would otherwise have to + # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the + # backfill checkpoint the ingester knows exactly (no parse_last_cursor). + run_stats: dict | None = None + cursor: str | None = None + posts_processed: int = 0 + # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so + # the platform cooldown matches the hint instead of a flat default. None when + # unknown (no header, or not a rate-limit failure). + retry_after_seconds: float | None = None + + +def extract_errors_warnings(stderr: str) -> str: + """Keep only the `[error]`/`[warning]` lines from a captured stderr blob. + + A generic download-log helper (module-level so the native-ingester result + path can shape its logs without reaching through a GalleryDLService instance). + """ + if not stderr: + return "" + kept = [ + line for line in stderr.splitlines() + if "][error]" in line.lower() or "][warning]" in line.lower() + ] + return "\n".join(kept) + + +def truncate_log(text: str, max_bytes: int = 500_000) -> str: + """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept).""" + if not text: + return text + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + half = max_bytes // 2 + head = encoded[:half].decode("utf-8", errors="ignore") + tail = encoded[-half:].decode("utf-8", errors="ignore") + head_lines = head.count("\n") + tail_lines = tail.count("\n") + total_lines = text.count("\n") + elided = max(0, total_lines - head_lines - tail_lines) + marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" + return head + marker + tail def _summarize_validation_failures(failures: list[dict]) -> str: @@ -164,19 +216,38 @@ def _summarize_validation_failures(failures: list[dict]) -> str: return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" -# gallery-dl logs its pagination cursor (when extractor.*.cursor is truthy) -# as `Cursor: ` — once per fetched page, at debug verbosity. The LAST -# such line is the furthest-progressed page, i.e. the resume point. We scan -# both streams (the debug log lands on stderr, but be defensive) and take the -# final match; passing it back as extractor.patreon.cursor resumes the walk. -# Survives a timed-out run too: the TimeoutExpired path returns the partial -# stdout/stderr, so an interrupted backfill still yields its last cursor. -_CURSOR_RE = re.compile(r"Cursor:\s*(\S+)") +# (parse_last_cursor was removed in plan #704: the native ingester now carries +# its checkpoint cursor as a structured DownloadResult.cursor field, so there is +# no log text to scrape — and gallery-dl platforms never had a cursor.) -def parse_last_cursor(stdout: str, stderr: str) -> str | None: - matches = _CURSOR_RE.findall(f"{stdout or ''}\n{stderr or ''}") - return matches[-1] if matches else None +def make_run_stats( + *, + exit_code: int = 0, + downloaded_count: int = 0, + skipped_count: int = 0, + per_item_failures: int = 0, + warning_count: int = 0, + tier_gated_count: int = 0, + quarantined_count: int = 0, + dead_lettered_count: int = 0, +) -> dict: + """The canonical `run_stats` dict shape, in ONE place. + + Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the + native ingester's per-outcome tally (ingest_core) — build through this so the + key set can't drift between backends. Phase 3 + the Logs UI read these keys. + """ + return { + "exit_code": exit_code, + "downloaded_count": downloaded_count, + "skipped_count": skipped_count, + "per_item_failures": per_item_failures, + "warning_count": warning_count, + "tier_gated_count": tier_gated_count, + "quarantined_count": quarantined_count, + "dead_lettered_count": dead_lettered_count, + } class GalleryDLService: @@ -212,16 +283,10 @@ class GalleryDLService: "permission denied", "tier required", "pledge required", ] - # Per-platform defaults. Lifted from GS — same six platforms FC supports. + # Per-platform defaults for the gallery-dl-backed platforms. Patreon was + # removed at the plan-#697 cutover — it now uses the native ingester + # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { - "patreon": { - "content_types": ["images", "image_large", "attachments", "postfile", "content"], - "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], - "filename": "{num:>02}_{filename}.{extension}", - "videos": True, - "embeds": True, - "cursor": True, - }, "subscribestar": { "content_types": ["all"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], @@ -306,25 +371,11 @@ class GalleryDLService: # transfer cap — large GIFs that keep streaming are unaffected. "retries": 2, "timeout": 60.0, - # Forward Patreon as Referer/Origin to yt-dlp when it - # fetches video manifests. Operator-flagged 2026-06-01 - # (DaferQ patreon): video posts hosted on Mux carry a JWT - # playback restriction that checks Referer/Origin on every - # request — not just the token signature. gallery-dl's - # HEAD probe to stream.mux.com returns 200 (the token is - # valid), but yt-dlp's actual GET-with-Range to fetch the - # m3u8 manifest 403s because yt-dlp sends its own default - # Referer, which Mux's policy rejects. Forcing the right - # headers fixes the headers-only case; Mux IP-range - # restrictions are unfixable from here. - "ytdl": { - "raw-options": { - "http_headers": { - "Referer": "https://www.patreon.com/", - "Origin": "https://www.patreon.com", - }, - }, - }, + # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived + # here until the plan-#697 cutover. It was Patreon-specific (and + # would have wrongly tagged the other platforms' yt-dlp fetches); + # Patreon video is now handled by the native ingester's + # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone. }, "output": {"progress": True}, } @@ -376,23 +427,7 @@ class GalleryDLService: platform_section = config["extractor"].setdefault(platform, {}) - if platform == "patreon": - if "all" in source_config.content_types: - platform_section["files"] = [ - "images", "image_large", "attachments", "postfile", "content", - ] - else: - platform_section["files"] = source_config.content_types - # Cursor-paged backfill: resume the newest→oldest walk from the - # checkpoint saved by the previous run instead of restarting from - # the top. Without this, a large catalog (Anduo's weekly - # image-dense Reports) never reaches its frontier inside the - # subprocess budget — the HEAD walk alone times out, 0 files - # written, no progress (event #40411). The PLATFORM_DEFAULTS leave - # `cursor: True` (log-only) for the first/fresh run. - if source_config.resume_cursor: - platform_section["cursor"] = source_config.resume_cursor - elif platform == "hentaifoundry": + if platform == "hentaifoundry": if "pictures" in source_config.content_types or "all" in source_config.content_types: platform_section["include"] = "all" @@ -552,10 +587,6 @@ class GalleryDLService: if not written_paths: return quarantined_relpaths, failures - quarantine_root = ( - self.images_root / "_quarantine" / artist_slug / platform - ) - for path in written_paths: if not is_validatable(path): continue @@ -566,34 +597,14 @@ class GalleryDLService: continue if result.ok: continue - try: - quarantine_root.mkdir(parents=True, exist_ok=True) - dest = quarantine_root / path.name - counter = 1 - while dest.exists(): - dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" - counter += 1 - path.rename(dest) - sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") - sidecar.write_text( - json.dumps( - { - "original_path": str(path), - "source_url": url, - "artist_slug": artist_slug, - "platform": platform, - "format": result.format, - "reason": result.reason, - "size": result.size, - "quarantined_at": datetime.now(UTC).isoformat(), - }, - indent=2, - ) - ) - except OSError as exc: - log.error("Failed to quarantine %s: %s. File left in place.", path, exc) - continue - + # Shared move+sidecar (file_validator.quarantine_file) — same impl the + # native ingester uses, so the layout + provenance sidecar can't drift. + dest = quarantine_file( + self.images_root, path, artist_slug, platform, + url=url, result=result, + ) + if dest is None: + continue # move failed → left in place, not counted log.warning( "Quarantined corrupt file: %s → %s (%s: %s)", path, dest, result.format, result.reason, @@ -631,41 +642,24 @@ class GalleryDLService: if "][warning]" in line.lower() and "not allowed to view post" in line.lower() ) - return { - "exit_code": return_code, - "downloaded_count": self._count_downloaded_files(stdout), - "skipped_count": skipped_stdout + skipped_stderr, - "per_item_failures": per_item_failures, - "warning_count": warning_count, - "tier_gated_count": tier_gated_count, - } + return make_run_stats( + exit_code=return_code, + downloaded_count=self._count_downloaded_files(stdout), + skipped_count=skipped_stdout + skipped_stderr, + per_item_failures=per_item_failures, + warning_count=warning_count, + tier_gated_count=tier_gated_count, + ) @staticmethod def _extract_errors_warnings(stderr: str) -> str: - if not stderr: - return "" - kept = [ - line for line in stderr.splitlines() - if "][error]" in line.lower() or "][warning]" in line.lower() - ] - return "\n".join(kept) + # Thin delegator to the module-level helper (kept for existing callers). + return extract_errors_warnings(stderr) @staticmethod def _truncate_log(text: str, max_bytes: int = 500_000) -> str: - if not text: - return text - encoded = text.encode("utf-8") - if len(encoded) <= max_bytes: - return text - half = max_bytes // 2 - head = encoded[:half].decode("utf-8", errors="ignore") - tail = encoded[-half:].decode("utf-8", errors="ignore") - head_lines = head.count("\n") - tail_lines = tail.count("\n") - total_lines = text.count("\n") - elided = max(0, total_lines - head_lines - tail_lines) - marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" - return head + marker + tail + # Thin delegator to the module-level helper (kept for existing callers). + return truncate_log(text, max_bytes) async def download( self, diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 213b86e..e1acc6e 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -31,7 +31,12 @@ from ..models import ( Source, ) from ..utils import safe_probe -from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name +from ..utils.paths import ( + derive_subdir, + derive_top_level_artist, + hash_suffixed_name, + safe_ext, +) from ..utils.phash import compute_phash, find_similar from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify @@ -82,27 +87,9 @@ def is_video(path: Path) -> bool: def _safe_ext(path: Path) -> str: """Conservatively extract a file extension for PostAttachment.ext - (varchar(32)). - - gallery-dl produces some filenames with URL-encoded query-string - artifacts embedded into the basename (e.g. - `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`). - `Path.suffix` finds the LAST dot and returns everything after, which - in those cases yields a 50+ char "extension" of mostly base64-ish - junk. That blows the column. Operator-flagged 2026-05-25. - - Real extensions are short and alphanumeric. We accept anything ≤ 16 - chars where every post-dot character is alphanumeric; anything else - means the input wasn't a real extension and we return the empty - string. ext is nullable-ish (empty string still satisfies NOT NULL) - and consumers should treat "" as "no known extension". - """ - suffix = path.suffix.lower() - if not suffix or len(suffix) > 16: - return "" - if not all(c.isalnum() for c in suffix[1:]): - return "" - return suffix + (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for + the Path-typed call sites + the [[path_suffix_sanitize]] memory pointer).""" + return safe_ext(path) def _mime_for(path: Path) -> str: diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py new file mode 100644 index 0000000..2b5286c --- /dev/null +++ b/backend/app/services/ingest_core.py @@ -0,0 +1,600 @@ +"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705). + +The orchestration that drives a native subscription walk — page a feed → +extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download → +mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped +`DownloadResult`, across tick/backfill/recovery modes — is identical for every +platform. Only four things are platform-specific, and they're INJECTED at +construction by a thin adapter (e.g. `PatreonIngester`): + + - `client` — `.iter_posts(feed_id, cursor)` yielding `(post, included, + page_cursor)` + `.extract_media(post, included) -> [media]`. + - `downloader`— `.download_post(post, media, artist_slug, is_seen) -> + [MediaOutcome]` (status in downloaded/skipped_seen/skipped_disk + /quarantined/error; `.path`/`.error`/`.post_id`). + - ledger — `seen_model` + `failed_model` SQLAlchemy models (+ their + on-conflict UNIQUE constraint names) and a `ledger_key(media)`. + - failure map — the adapter overrides `_failure_result` (platform exception + → DownloadResult.error_type) and supplies `error_base` (the + exception type the walk catches) + `platform` (result label). + +Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker — +never held across a network fetch ([[db-connection-held-across-subprocess]]). +Plain-HTTP homelab: no secure-context Web API. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable + +from sqlalchemy import delete, func, select, text +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from .gallery_dl import DownloadResult, ErrorType, make_run_stats + +log = logging.getLogger(__name__) + +# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or +# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of +# per-file HEADs. Headroom against paywalled/undownloadable items interleaving. +_TICK_SEEN_THRESHOLD = 20 + +# plan #705 #7: after this many failed download/validate attempts a media is +# "dead-lettered" and skipped on routine tick/backfill walks (recovery still +# re-attempts it). Stops a permanently-broken media re-erroring forever. +DEAD_LETTER_THRESHOLD = 3 +# last_error is Text but bound it so a giant traceback doesn't bloat the row. +_ERROR_MAX = 1000 + + +class Ingester: + """Generic native-ingest orchestration. Subclass with a platform adapter + (see the module docstring) — or construct directly with the keyword seams.""" + + def __init__( + self, + *, + client, + downloader, + session_factory: Callable[[], object], + seen_model, + failed_model, + seen_constraint: str, + failed_constraint: str, + ledger_key: Callable[[object], str], + platform: str, + error_base: type[Exception], + ): + self.client = client + self.downloader = downloader + self.session_factory = session_factory + self._seen_model = seen_model + self._failed_model = failed_model + self._seen_constraint = seen_constraint + self._failed_constraint = failed_constraint + self._ledger_key = ledger_key + self._platform = platform + self._error_base = error_base + + # -- public ------------------------------------------------------------ + + def run( + self, + *, + source_id: int, + campaign_id: str, + artist_slug: str, + url: str, + mode: str, + resume_cursor: str | None = None, + time_budget_seconds: float = 870.0, + seen_threshold: int = _TICK_SEEN_THRESHOLD, + posts_base: int = 0, + ) -> DownloadResult: + """Walk + download for one source, returning a gallery-dl-shaped result. + + `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1 + seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept + files). The walk stops on: + - budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL + - tick early-out (seen_threshold contiguous seen) → success + - reaching the bottom of the feed → success (rc 0) + A client-level failure (drift / auth / network) fails the whole run loud. + """ + bypass_seen = mode == "recovery" + # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a + # tick has no resumable backfill state. + checkpoint = mode in ("backfill", "recovery") + ledger_key = self._ledger_key + start = time.monotonic() + log_lines: list[str] = [] + written: list[str] = [] + quarantined_paths: list[str] = [] + downloaded = 0 + errors = 0 + quarantined = 0 + dead_lettered = 0 + skipped_count = 0 + posts_processed = 0 + # Net-new posts THIS chunk for the live progress badge (plan #704 #5); + # excludes the re-walked resume page so _backfill_posts stays a monotonic + # absolute across chunks instead of an inflating sum. posts_processed + # stays the gross per-chunk count used for the run summary. + chunk_new_posts = 0 + consecutive_seen = 0 + emitted_cursor: str | None = None + reached_bottom = False + budget_hit = False + early_out = False + stopped = False # plan #708 B4: operator hit Stop mid-walk + cancel_armed = False # latched once we observe a live "running" state + + def _result( + *, success: bool, return_code: int, + error_type: ErrorType | None, error_message: str | None, + ) -> DownloadResult: + # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor + # directly instead of regex-scraping a reconstructed stdout. stdout + # stays a human-readable summary (no fake `Cursor:` lines). + return DownloadResult( + success=success, + url=url, + artist_slug=artist_slug, + platform=self._platform, + files_downloaded=downloaded, + files_quarantined=quarantined, + quarantined_paths=list(quarantined_paths), + written_paths=written, + stdout="\n".join(log_lines), + stderr="", + return_code=return_code, + error_type=error_type, + error_message=error_message, + duration_seconds=time.monotonic() - start, + cursor=emitted_cursor, + posts_processed=posts_processed, + run_stats=make_run_stats( + exit_code=return_code, + downloaded_count=downloaded, + skipped_count=skipped_count, + per_item_failures=errors, + quarantined_count=quarantined, + dead_lettered_count=dead_lettered, + ), + ) + + try: + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=resume_cursor + ): + # Checkpoint the cursor that FETCHED this page the moment we + # START it — so a chunk cut mid-page resumes the page, not the one + # after it. Carried as DownloadResult.cursor (plan #704). + if page_cursor and page_cursor != emitted_cursor: + emitted_cursor = page_cursor + # plan #705 #6: persist the cursor at each page boundary so a + # worker SIGKILL mid-chunk resumes near the crash, not the + # chunk start. (phase 3 still writes the final cursor — same + # value; this is the crash-safety net.) plan #704 #5: persist + # the live posts count alongside it so the badge climbs DURING + # the chunk, not only when it ends. + if checkpoint: + # plan #708 B4: an operator Stop pops `_backfill_state` — + # bail at the page boundary (progress already checkpointed) + # before more network work, so the live chunk halts + # promptly instead of running to its time-box. LATCH on the + # first observed "running" state, so a run invoked WITHOUT a + # running state (a unit test, or a stale call) never + # spuriously self-cancels. A short SELECT, never held. + if self._still_running(source_id): + cancel_armed = True + elif cancel_armed: + stopped = True + break + self._checkpoint_cursor(source_id, emitted_cursor) + self._checkpoint_posts(source_id, posts_base + chunk_new_posts) + + # Time-box check at the post boundary (coarse, like a gallery-dl + # chunk). Backfill/recovery resume from emitted_cursor next chunk. + if time.monotonic() - start >= time_budget_seconds: + budget_hit = True + break + + posts_processed += 1 + # The resume page (its cursor == resume_cursor) was already + # counted by the chunk that checkpointed it — don't re-count it + # into the persisted badge (plan #704 #5). First chunk has + # resume_cursor None, so everything counts. + if not (resume_cursor and page_cursor == resume_cursor): + chunk_new_posts += 1 + media = self.client.extract_media(post, included) + if not media: + continue + + keys = [ledger_key(m) for m in media] + # Recovery bypasses BOTH the seen-ledger AND the dead-letter + # ledger (the operator's "try everything again"); routine walks + # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7). + dead = set() if bypass_seen else self._dead_keys(source_id, keys) + seen = ( + set() + if bypass_seen + else self._seen_keys(source_id, keys) + ) + skip = seen | dead + + def _is_skip(m, _skip=skip) -> bool: + return ledger_key(m) in _skip + + outcomes = self.downloader.download_post( + post, media, artist_slug, is_seen=_is_skip + ) + + to_mark: list[tuple[str, str]] = [] + to_clear: list[str] = [] # recovered → drop any dead-letter row + to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error) + for media_item, outcome in zip(media, outcomes, strict=False): + key = ledger_key(media_item) + if key in dead: + dead_lettered += 1 # skipped because previously dead + if outcome.status == "downloaded": + downloaded += 1 + if outcome.path is not None: + written.append(str(outcome.path)) + to_mark.append((key, media_item.post_id)) + to_clear.append(key) + consecutive_seen = 0 + elif outcome.status == "skipped_disk": + # Already on disk (a prior run). Reconcile the ledger so a + # later tick skips it at tier-1 without a disk stat, but + # do NOT re-feed it to phase 3 — attach_in_place would see + # the duplicate sha256 and unlink the on-disk copy. + to_mark.append((key, media_item.post_id)) + to_clear.append(key) + skipped_count += 1 + consecutive_seen += 1 + elif outcome.status == "skipped_seen": + skipped_count += 1 + consecutive_seen += 1 + elif outcome.status == "quarantined": + # New content that failed validation (corrupt) — counted + # distinctly so the run surfaces a real quarantined total. + # Not marked seen (a later walk may re-fetch a fixed file); + # it IS new content, so it breaks the run-of-seen. Counts + # toward the dead-letter ledger (plan #705 #7). + quarantined += 1 + if outcome.path is not None: + quarantined_paths.append(str(outcome.path)) + to_fail.append((key, media_item.post_id, outcome.error or "quarantined")) + consecutive_seen = 0 + elif outcome.status == "error": + errors += 1 + to_fail.append((key, media_item.post_id, outcome.error or "error")) + # An error neither advances nor resets the run-of-seen. + + if mode == "tick" and consecutive_seen >= seen_threshold: + early_out = True + break + + # Persist ledger changes AFTER the network fetch, on short + # sessions: mark downloaded/on-disk seen, clear any dead-letter + # for recovered media, and record failures (plan #705 #7). + if to_mark: + self._mark_seen(source_id, to_mark) + if to_clear: + self._clear_failures(source_id, to_clear) + if to_fail: + self._record_failures(source_id, to_fail) + + if early_out: + break + else: + reached_bottom = True + except self._error_base as exc: + # The platform's client-error base — _failure_result (adapter) + # maps it to a typed error. + return self._failure_result(exc, _result) + + # plan #708 B4: a Stop already popped the backfill state (incl. cursor + + # posts), so don't re-write them — return PARTIAL (reads as "ok/progress", + # the lifecycle no-ops since state is gone) instead of a false "complete". + if stopped: + return _result( + success=False, return_code=-1, + error_type=ErrorType.PARTIAL, + error_message=f"Stopped by operator: {downloaded} file(s) this chunk", + ) + + # Final authoritative posts count for the badge — captures the last page + # after the last boundary write and the time-box break (plan #704 #5). + if checkpoint: + self._checkpoint_posts(source_id, posts_base + chunk_new_posts) + + if errors: + log_lines.append(f"{errors} media item(s) failed") + if quarantined: + log_lines.append(f"{quarantined} media item(s) quarantined (invalid)") + if dead_lettered: + log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)") + log_lines.append( + f"{self._platform} ingest ({mode}): {downloaded} downloaded, " + f"{skipped_count} skipped, {quarantined} quarantined, " + f"{dead_lettered} dead-lettered, {errors} error(s), " + f"{posts_processed} post(s)" + + (", reached end" if reached_bottom else "") + + (", time-boxed" if budget_hit else "") + ) + + if budget_hit: + # A chunk that hit its time-box but made forward progress is a + # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the + # next chunk resumes from the emitted cursor. No progress → TIMEOUT, + # which feeds download_service's backfill stall-guard. rc<0 mirrors + # subprocess TimeoutExpired so completion detection stays false. + made_progress = downloaded > 0 or emitted_cursor != resume_cursor + if made_progress: + return _result( + success=False, return_code=-1, + error_type=ErrorType.PARTIAL, + error_message=( + f"Backfill chunk: {downloaded} file(s) — continuing" + ), + ) + return _result( + success=False, return_code=-1, + error_type=ErrorType.TIMEOUT, + error_message="Chunk timed out with no progress", + ) + + # Normal success: reached the bottom, or a tick that early-outed. rc 0 + + # error_type None is REQUIRED for a backfill/recovery walk that reached + # the bottom to be marked COMPLETE by + # download_service._apply_backfill_lifecycle — so we return None even + # when downloaded == 0 (a re-confirming walk that found nothing new still + # completed). success=True maps to status "ok" regardless. A tick that + # early-outed also returns here; ticks never set backfill state so the + # lifecycle is a no-op for them. + return _result( + success=True, return_code=0, + error_type=None, error_message=None, + ) + + # -- preview (dry-run) ------------------------------------------------- + + def preview( + self, + source_id: int, + campaign_id: str, + *, + page_limit: int = 3, + sample_size: int = 10, + ) -> dict: + """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media + NOT already in the seen/dead ledgers, WITHOUT downloading anything. + + Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets + an operator gauge "is this source worth a backfill?" cheaply. Returns: + {total_new, posts_scanned, pages_scanned, has_more, + sample: [{title, date, new}, ...]} # sample = posts with new media + A client-level failure (auth/drift) propagates to the caller. + """ + total_new = 0 + posts_scanned = 0 + pages_scanned = 0 + has_more = False + sample: list[dict] = [] + unset = object() + last_page: object = unset + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=None + ): + if page_cursor != last_page: + last_page = page_cursor + pages_scanned += 1 + if pages_scanned > page_limit: + has_more = True + pages_scanned = page_limit + break + posts_scanned += 1 + media = self.client.extract_media(post, included) + if not media: + continue + keys = [self._ledger_key(m) for m in media] + skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys) + new_count = sum(1 for m in media if self._ledger_key(m) not in skip) + total_new += new_count + if new_count > 0 and len(sample) < sample_size: + meta = self.client.post_meta(post) + sample.append( + { + "title": meta.get("title") or "(untitled)", + "date": meta.get("date"), + "new": new_count, + } + ) + return { + "total_new": total_new, + "posts_scanned": posts_scanned, + "pages_scanned": pages_scanned, + "has_more": has_more, + "sample": sample, + } + + # -- failure mapping (adapter overrides) ------------------------------- + + def _failure_result(self, exc: Exception, _result) -> DownloadResult: + """Map a platform client-error to a typed failed DownloadResult. The base + gives a safe default; adapters override with their exception taxonomy.""" + log.warning("%s ingest failed: %s", self._platform, exc) + return _result( + success=False, return_code=1, + error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc), + ) + + # -- seen-ledger (short-lived sessions) -------------------------------- + + def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` are already in the seen-ledger for this source. + + One short SELECT on its own session — opened and closed without any + network in between (the GETs happen after, in download_post). + """ + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(self._seen_model.filehash).where( + self._seen_model.source_id == source_id, + self._seen_model.filehash.in_(keys), + ) + ).scalars().all() + return set(rows) + + def _checkpoint_cursor(self, source_id: int, cursor: str) -> None: + """Persist the in-progress backfill cursor mid-walk (plan #705 #6). + + ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just + `_backfill_cursor`, cast back — so it never clobbers operator config or + the other backfill keys (no read-modify-write race). The in-flight guard + means only this source's one download runs at a time; a concurrent + operator stop is benign (a stray cursor with no `_backfill_state` is + ignored by tick mode and cleared on the next start). + """ + with self.session_factory() as session: + session.execute( + text( + "UPDATE source SET config_overrides = jsonb_set(" + " coalesce(config_overrides::jsonb, '{}'::jsonb)," + " '{_backfill_cursor}', to_jsonb(cast(:cur AS text))" + ")::json WHERE id = :sid" + ), + {"cur": cursor, "sid": source_id}, + ) + session.commit() + + def _still_running(self, source_id: int) -> bool: + """True while the source is armed for a deep walk (plan #708 B4). + + An operator Stop (`source_service.stop_backfill`) pops `_backfill_state`, + so a False here means "cancel this chunk now". One short SELECT on its own + session — never held across the walk + ([[db-connection-held-across-subprocess]]).""" + with self.session_factory() as session: + state = session.execute( + text( + "SELECT config_overrides::jsonb ->> '_backfill_state' " + "FROM source WHERE id = :sid" + ), + {"sid": source_id}, + ).scalar_one_or_none() + return state == "running" + + def _checkpoint_posts(self, source_id: int, posts: int) -> None: + """Persist the live backfill posts-processed count mid-walk (plan #704 #5). + + Same atomic single-key jsonb_set dance as _checkpoint_cursor, on the + `_backfill_posts` key (cast to a JSON number) — so the progress badge + climbs DURING a chunk without clobbering operator config or the cursor. + The ingester OWNS this key now; download_service no longer accumulates it + post-chunk (which lagged a whole chunk and over-counted the resume page). + """ + with self.session_factory() as session: + session.execute( + text( + "UPDATE source SET config_overrides = jsonb_set(" + " coalesce(config_overrides::jsonb, '{}'::jsonb)," + " '{_backfill_posts}', to_jsonb(cast(:posts AS int))" + ")::json WHERE id = :sid" + ), + {"posts": posts, "sid": source_id}, + ) + session.commit() + + def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: + """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page. + + ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a + re-sighting — or a concurrent walk — is a harmless no-op + ([[scalar_one_or_none-duplicates]]: never check-then-insert without the + DB constraint backing it). De-dup the batch locally first so a single + page can't present the same key twice to one INSERT. + """ + seen_local: set[str] = set() + values = [] + for key, post_id in items: + if key in seen_local: + continue + seen_local.add(key) + values.append( + {"source_id": source_id, "filehash": key, "post_id": post_id} + ) + if not values: + return + with self.session_factory() as session: + stmt = pg_insert(self._seen_model).values(values) + stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint) + session.execute(stmt) + session.commit() + + # -- dead-letter ledger (plan #705 #7) --------------------------------- + + def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead). + One short SELECT; recovery never calls this (it re-attempts dead media).""" + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(self._failed_model.filehash).where( + self._failed_model.source_id == source_id, + self._failed_model.filehash.in_(keys), + self._failed_model.attempts >= DEAD_LETTER_THRESHOLD, + ) + ).scalars().all() + return set(rows) + + def _record_failures( + self, source_id: int, items: list[tuple[str, str, str]] + ) -> None: + """Upsert-increment the dead-letter ledger for failed media. On conflict + bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the + upsert — no check-then-insert). De-dup the batch (one row/key, last error + wins).""" + by_key: dict[str, str] = {} + for key, _post_id, err in items: + by_key[key] = (err or "")[:_ERROR_MAX] + if not by_key: + return + values = [ + {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e} + for k, e in by_key.items() + ] + with self.session_factory() as session: + stmt = pg_insert(self._failed_model).values(values) + stmt = stmt.on_conflict_do_update( + constraint=self._failed_constraint, + set_={ + "attempts": self._failed_model.attempts + 1, + "last_error": stmt.excluded.last_error, + "last_failed_at": func.now(), + }, + ) + session.execute(stmt) + session.commit() + + def _clear_failures(self, source_id: int, keys: list[str]) -> None: + """Drop dead-letter rows for media that just downloaded cleanly — they + recovered. A no-op DELETE for keys that were never failing.""" + unique = list(dict.fromkeys(keys)) + if not unique: + return + with self.session_factory() as session: + session.execute( + delete(self._failed_model).where( + self._failed_model.source_id == source_id, + self._failed_model.filehash.in_(unique), + ) + ) + session.commit() diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py new file mode 100644 index 0000000..98d8134 --- /dev/null +++ b/backend/app/services/patreon_client.py @@ -0,0 +1,610 @@ +"""Native Patreon JSON:API client (build step 1 of the native ingester). + +Clean-room reimplementation of the Patreon `/api/posts` read path. This is a +plain, synchronous client over `requests` — the orchestrator already wraps +sync importer calls, so nothing here needs to be async (mirrors +patreon_resolver.py, which wraps its sync lookup in run_in_executor at the +call site). + +Scope (build step 1): fetch + page + parse only. This module is NOT wired into +download_service yet — that is a later step. The public surface here exists so +the later step can drive it: + - PatreonClient(cookies_path).iter_posts(campaign_id) + → (post, included_index, page_cursor) + - extract_media(post, included_index) → list[MediaItem] + - parse_cursor_from_url(url) → cursor + +Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we +depend on (top-level `data`, media resources carrying `file_name`/`url`) are +the contract. If a response comes back as an HTML login page or a media +resource is missing the fields we resolve against, we raise PatreonDriftError +rather than silently yielding empty media — so the later import step surfaces +"Patreon changed something" instead of "creator has no posts". + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import http.cookiejar +import logging +import os +import re +import time +from collections.abc import Iterator +from dataclasses import dataclass +from html import unescape +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import requests + +from ..utils.paths import safe_ext + +log = logging.getLogger(__name__) + +_POSTS_URL = "https://www.patreon.com/api/posts" +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +) +_TIMEOUT_SECONDS = 30.0 + +# 429 backoff (plan #703): ride out a transient API rate-limit instead of +# failing the whole walk (which would stamp RATE_LIMITED → platform-wide +# cooldown → every Patreon source dark). Honor the server's `Retry-After`; +# otherwise exponential base·2^(n-1), capped. Only after the retries are +# exhausted does the 429 propagate as terminal RATE_LIMITED. +_MAX_429_RETRIES = 3 +_BACKOFF_BASE_SECONDS = 2.0 +_BACKOFF_CAP_SECONDS = 30.0 + + +def _retry_after_seconds( + resp: requests.Response, + attempt: int, + *, + base: float = _BACKOFF_BASE_SECONDS, + cap: float = _BACKOFF_CAP_SECONDS, +) -> float: + """Backoff delay for a 429: the `Retry-After` seconds header if present and + numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form + of Retry-After is rare here and falls through to exponential.)""" + header = resp.headers.get("Retry-After") + if header: + try: + return min(float(header), cap) + except (TypeError, ValueError): + pass + return min(base * (2 ** max(0, attempt - 1)), cap) + +# JSON:API request contract (observed from real traffic — see module plan). +_INCLUDE = ( + "campaign,access_rules,attachments,attachments_media,audio,images,media," + "native_video_insights,user,user_defined_tags,ti_checks" +) +_FIELDS_POST = ( + "content,post_file,image,post_type,published_at,title,url,patreon_url," + "current_user_can_view" +) +_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" +_FIELDS_CAMPAIGN = "name,url" + +# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is +# Patreon's stable per-file identity and is what we dedup + ledger against. +# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere +# in the URL (path or query); real Patreon CDN URLs carry exactly one. +_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") + +# Inline post `content` is HTML; images are emitted as . +# Pull every src; downstream dedup collapses any that duplicate a gallery item +# by filehash. Tolerant of attribute ordering and single/double quotes. +_CONTENT_IMG_RE = re.compile(r"]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE) + + +class PatreonAPIError(Exception): + """Base for native Patreon client failures. + + `status_code` carries the HTTP status when the failure was an HTTP response + (None for transport-level / parse failures), so the ingester can map it to a + DownloadResult.error_type (429 → rate_limited, 404 → not_found, …). + `retry_after` carries the server's `Retry-After` seconds on a terminal 429, so + the platform cooldown can match the server's hint instead of a flat default + (plan #708 B1). + """ + + def __init__( + self, + message: str, + *, + status_code: int | None = None, + retry_after: float | None = None, + ): + super().__init__(message) + self.status_code = status_code + self.retry_after = retry_after + + +class PatreonAuthError(PatreonAPIError): + """Authentication / authorization failure — missing or expired session + cookies, an insufficient pledge tier, or an HTML login/challenge page served + where JSON was expected. DISTINCT from drift: the fix is rotating the + credential, not updating the ingester. Maps to error_type 'auth_error'. + """ + + +class PatreonDriftError(PatreonAPIError): + """A JSON response did not match the JSON:API shape we depend on. + + Raised for: a missing top-level `data` list, `data` not a list, or a media + resource lacking the `file_name`/`url` fields we resolve against. Fail loud + so the import step flags API drift (ingester needs update) instead of + silently importing nothing. An HTML-login / non-JSON body is auth, not + drift — that raises PatreonAuthError. + """ + + +@dataclass +class MediaItem: + """One resolved downloadable item belonging to a post. + + Fields: + url — the CDN/download URL to fetch. + filename — media `file_name` when present; otherwise the URL basename + (NEVER a network call). Bounded/sane extension. + kind — one of: "images", "image_large", "attachments", "postfile", + "content". Mirrors gallery-dl's `files` content-type names so + the later step can honor the same per-source content_types. + filehash — the 32-char hex (MD5) segment from the CDN URL, or None if + the URL carries no such segment. Used for in-post dedup and + the cross-run seen-ledger. + post_id — the owning post's id (so a flattened media list stays + traceable to its post). + """ + + url: str + filename: str + kind: str + filehash: str | None + post_id: str + + +def _load_session(cookies_path: str | Path | None) -> requests.Session: + session = requests.Session() + session.headers.update( + { + "User-Agent": _USER_AGENT, + "Accept": "application/vnd.api+json", + } + ) + if cookies_path and os.path.isfile(str(cookies_path)): + try: + jar = http.cookiejar.MozillaCookieJar(str(cookies_path)) + jar.load(ignore_discard=True, ignore_expires=True) + session.cookies = jar # type: ignore[assignment] + except (OSError, http.cookiejar.LoadError) as exc: + log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc) + return session + + +def _filehash(url: str) -> str | None: + if not url: + return None + match = _FILEHASH_RE.search(url) + return match.group(1).lower() if match else None + + +def _basename_from_url(url: str) -> str: + """Derive a sane filename from a URL when the media has no file_name. + + Strips query/fragment, takes the path basename, and drops a junk + extension (the importer._safe_ext gotcha) so we never write base64 noise + as a name. Falls back to the filehash, then to "file". + """ + path = urlsplit(url).path + base = os.path.basename(path) + if base: + ext = safe_ext(base) + stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base + # Keep the stem bounded; URL-encoded stems can be enormous. + stem = stem[:120] or "file" + return f"{stem}{ext}" + fh = _filehash(url) + return fh or "file" + + +def parse_cursor_from_url(url: str | None) -> str | None: + """Extract the `page[cursor]` query param from a links.next URL.""" + if not url: + return None + query = urlsplit(url).query + values = parse_qs(query).get("page[cursor]") + if values and values[0]: + return values[0] + return None + + +class PatreonClient: + """Synchronous Patreon JSON:API read client. + + Construct with a path to a Netscape cookies.txt (the same file + CredentialService.get_cookies_path materializes). Cookies are loaded into a + requests.Session; no secure-context APIs are used. + """ + + def __init__( + self, + cookies_path: str | Path | None, + *, + request_sleep: float = 0.0, + max_retries: int = _MAX_429_RETRIES, + ): + self.cookies_path = str(cookies_path) if cookies_path else None + self._session = _load_session(cookies_path) + # Politeness: seconds to sleep before each /api/posts page fetch (paces + # the rate-limited API endpoint). 0 = no pacing. plan #703. + self._request_sleep = request_sleep or 0.0 + self._max_retries = max_retries + + # -- request ----------------------------------------------------------- + + def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]: + params = { + "include": _INCLUDE, + "fields[post]": _FIELDS_POST, + "fields[media]": _FIELDS_MEDIA, + "fields[campaign]": _FIELDS_CAMPAIGN, + "filter[campaign_id]": campaign_id, + "filter[contains_exclusive_posts]": "true", + "filter[is_draft]": "false", + "sort": "-published_at", + "json-api-version": "1.0", + } + if cursor: + params["page[cursor]"] = cursor + return params + + def _fetch(self, campaign_id: str, cursor: str | None) -> dict: + if self._request_sleep > 0: + time.sleep(self._request_sleep) # pace the API endpoint + attempt = 0 + while True: + try: + resp = self._session.get( + _POSTS_URL, + params=self._params(campaign_id, cursor), + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + raise PatreonAPIError( + f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" + ) from exc + + # Transient rate-limit: back off and retry rather than failing the + # whole walk. Only a PERSISTENT 429 (retries exhausted) falls + # through to the terminal RATE_LIMITED raise below. + if resp.status_code == 429 and attempt < self._max_retries: + attempt += 1 + delay = _retry_after_seconds(resp, attempt) + log.warning( + "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)", + campaign_id, delay, attempt, self._max_retries, + ) + time.sleep(delay) + continue + break + + if resp.status_code in (401, 403): + # Auth rejected — expired/missing cookies or an insufficient tier. + # Actionable as "rotate credentials", so it's auth, not drift/http. + raise PatreonAuthError( + f"Patreon posts API returned HTTP {resp.status_code} — auth " + f"rejected (cookies expired or tier insufficient; " + f"campaign_id={campaign_id})", + status_code=resp.status_code, + ) + if resp.status_code != 200: + # A persistent 429 (retries exhausted) is terminal RATE_LIMITED — carry + # the server's raw Retry-After seconds so the cooldown matches its hint + # (plan #708 B1). Header is uncapped here; the cooldown clamps it. + retry_after = None + if resp.status_code == 429: + hdr = resp.headers.get("Retry-After") + if hdr: + try: + retry_after = float(hdr) + except (TypeError, ValueError): + retry_after = None + raise PatreonAPIError( + f"Patreon posts API returned HTTP {resp.status_code} " + f"(campaign_id={campaign_id})", + status_code=resp.status_code, + retry_after=retry_after, + ) + try: + payload = resp.json() + except ValueError as exc: + # A non-JSON body here is almost always the HTML login/challenge + # page served when cookies are missing/expired — that is an AUTH + # failure (rotate cookies), not API drift (update the ingester) and + # not a transient network error. + raise PatreonAuthError( + "Patreon posts API returned a non-JSON response (likely an " + f"HTML login/challenge page — session expired; " + f"campaign_id={campaign_id}): {exc}" + ) from exc + return payload + + # -- parsing ----------------------------------------------------------- + + @staticmethod + def _transform(response: dict) -> dict: + """Flatten the JSON:API `included` array for relationship resolution. + + Returns a dict keyed by `(type, id)` → that resource's `attributes` + (so a post's relationships can be resolved in O(1)). Missing/oddly + shaped `included` entries are skipped rather than fatal — drift + detection for the top-level shape lives in _validate_response. + """ + index: dict[tuple[str, str], dict] = {} + for inc in response.get("included") or []: + if not isinstance(inc, dict): + continue + rtype = inc.get("type") + rid = inc.get("id") + if rtype is None or rid is None: + continue + index[(str(rtype), str(rid))] = inc.get("attributes") or {} + return index + + @staticmethod + def _validate_response(response: dict) -> None: + if not isinstance(response, dict): + raise PatreonDriftError("Patreon response was not a JSON object") + if "data" not in response: + raise PatreonDriftError("Patreon response missing top-level 'data' key") + data = response.get("data") + if not isinstance(data, list): + raise PatreonDriftError("Patreon response 'data' was not a list") + + def _related_ids(self, post: dict, rel_name: str) -> list[str]: + rels = post.get("relationships") or {} + rel = rels.get(rel_name) or {} + data = rel.get("data") + if isinstance(data, dict): # to-one relationship + data = [data] + if not isinstance(data, list): + return [] + ids: list[str] = [] + for ref in data: + if isinstance(ref, dict) and ref.get("id") is not None: + ids.append(str(ref["id"])) + return ids + + @staticmethod + def _media_url(attrs: dict) -> str | None: + """Pick the best fetchable URL for a media resource. + + Prefer the full-size `download_url`; fall back to the largest + `image_urls` size. gallery-dl prefers download_url too, only dipping + into image_urls when a smaller configured size is requested — FC + always wants the original, so download_url first. + """ + download_url = attrs.get("download_url") + if isinstance(download_url, str) and download_url: + return download_url + image_urls = attrs.get("image_urls") + if isinstance(image_urls, dict): + for key in ("original", "full", "large", "default"): + candidate = image_urls.get(key) + if isinstance(candidate, str) and candidate: + return candidate + # Otherwise take any non-empty string value. + for candidate in image_urls.values(): + if isinstance(candidate, str) and candidate: + return candidate + return None + + def _media_item( + self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool + ) -> MediaItem: + url = self._media_url(attrs) + if not url: + raise PatreonDriftError( + f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL " + f"(no download_url / image_urls)" + ) + file_name = attrs.get("file_name") + if require_file_name and not (isinstance(file_name, str) and file_name): + raise PatreonDriftError( + f"Patreon media resource (post {post_id}, kind={kind}) missing " + f"'file_name'" + ) + filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url) + return MediaItem( + url=url, + filename=filename, + kind=kind, + filehash=_filehash(url), + post_id=post_id, + ) + + def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]: + """Resolve all downloadable media for one post. + + Walks the kinds in the same order gallery-dl does — images, + image_large (post cover), attachments, postfile, content (inline + ) — and dedups within the post by filehash (first wins). The + image_large cover commonly duplicates a gallery image; deduping by + filehash collapses them to the gallery item (encountered first). + """ + post_id = str(post.get("id") or "") + attrs = post.get("attributes") or {} + items: list[MediaItem] = [] + + def _resolve_rel(rel_name: str, kind: str) -> None: + for mid in self._related_ids(post, rel_name): + media_attrs = included_index.get(("media", mid)) + if media_attrs is None: + # Referenced but not in `included`: a media id with no + # resource is drift (we asked for include=media). + raise PatreonDriftError( + f"Patreon post {post_id} references media {mid} " + f"({rel_name}) not present in 'included'" + ) + items.append( + self._media_item( + media_attrs, kind, post_id, require_file_name=True + ) + ) + + # 1. gallery images + _resolve_rel("images", "images") + + # 2. image_large — the post-level cover (`image.large_url`). Not a + # media relationship; it lives on the post attributes. + image = attrs.get("image") + if isinstance(image, dict): + large_url = image.get("large_url") or image.get("url") + if isinstance(large_url, str) and large_url: + items.append( + MediaItem( + url=large_url, + filename=_basename_from_url(large_url), + kind="image_large", + filehash=_filehash(large_url), + post_id=post_id, + ) + ) + + # 3. attachments + _resolve_rel("attachments_media", "attachments") + + # 4. postfile — the post's primary attached file (`post_file`). + post_file = attrs.get("post_file") + if isinstance(post_file, dict): + pf_url = post_file.get("url") or post_file.get("download_url") + if isinstance(pf_url, str) and pf_url: + pf_name = post_file.get("name") + filename = ( + pf_name + if isinstance(pf_name, str) and pf_name + else _basename_from_url(pf_url) + ) + items.append( + MediaItem( + url=pf_url, + filename=filename, + kind="postfile", + filehash=_filehash(pf_url), + post_id=post_id, + ) + ) + + # 5. content — inline in the post HTML body. + content = attrs.get("content") + if isinstance(content, str) and content: + for raw_src in _CONTENT_IMG_RE.findall(content): + src = unescape(raw_src) + if not src: + continue + items.append( + MediaItem( + url=src, + filename=_basename_from_url(src), + kind="content", + filehash=_filehash(src), + post_id=post_id, + ) + ) + + return _dedup_by_filehash(items) + + @staticmethod + def post_meta(post: dict) -> dict: + """Title + published date for a post — for the preview sample (plan #708 + B4). Part of the client contract `ingest_core.Ingester.preview` calls.""" + attrs = post.get("attributes") or {} + title = attrs.get("title") + published = attrs.get("published_at") + return { + "title": title if isinstance(title, str) else None, + "date": published if isinstance(published, str) else None, + } + + # -- iteration --------------------------------------------------------- + + def iter_posts( + self, campaign_id: str, cursor: str | None = None + ) -> Iterator[tuple[dict, dict, str | None]]: + """Yield (post, included_index, page_cursor) for every post in the feed. + + Pages newest→oldest via `links.next`, validating each response for + drift before yielding. The triple gives a caller everything it needs to + resolve and checkpoint without re-fetching: + - post — the raw post resource. + - included_index — the page's flattened `included` (the same object + for every post on a page), to pass straight to + extract_media(post, included_index). + - page_cursor — the cursor that FETCHED this post's page (None for + the first page). The caller checkpoints THIS value, + matching the existing backfill cursor logic where + the saved cursor re-fetches the page being + processed (so a chunk cut mid-page resumes the page, + not the one after it). + """ + current_cursor = cursor + while True: + response = self._fetch(campaign_id, current_cursor) + self._validate_response(response) + page_cursor = current_cursor + included_index = self._transform(response) + for post in response.get("data") or []: + if isinstance(post, dict): + yield post, included_index, page_cursor + next_url = (response.get("links") or {}).get("next") + next_cursor = parse_cursor_from_url(next_url) + if not next_cursor: + return + current_cursor = next_cursor + + # -- verify ------------------------------------------------------------ + + def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: + """Cheap auth probe: fetch the first `/api/posts` page and report whether + the credential authenticated, WITHOUT downloading anything. + + Returns `(ok, message)` matching the credential-verify contract: + - True — authenticated (the feed returned a valid JSON:API page). + - False — the credential was rejected (PatreonAuthError: 401/403, or an + HTML login page → cookies expired / tier insufficient). + - None — inconclusive: API drift (our parser is stale, not a cred + problem) or a transient network/HTTP error. + """ + try: + response = self._fetch(campaign_id, None) + self._validate_response(response) + except PatreonAuthError as exc: + return False, f"Patreon rejected the credential — {exc}" + except PatreonDriftError as exc: + return None, f"Couldn't verify — Patreon's API shape changed: {exc}" + except PatreonAPIError as exc: + return None, f"Couldn't verify (network/HTTP issue): {exc}" + return True, "Credentials valid — the Patreon feed authenticated." + + +def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]: + """Drop later items sharing a filehash with an earlier one (first wins). + + Items with no filehash (None) are never deduped against each other — we + can't prove they're the same file, so keep them all. + """ + seen: set[str] = set() + out: list[MediaItem] = [] + for item in items: + if item.filehash is not None: + if item.filehash in seen: + continue + seen.add(item.filehash) + out.append(item) + return out diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py new file mode 100644 index 0000000..b5a9dea --- /dev/null +++ b/backend/app/services/patreon_downloader.py @@ -0,0 +1,501 @@ +"""Native Patreon media downloader (build step 2b of the native ingester). + +Given a Patreon post and its already-resolved `MediaItem`s (from +patreon_client.extract_media), download the media to the EXACT on-disk layout +gallery-dl produces, write a sidecar JSON the existing importer consumes, and +report per-media outcomes. + +This module is PURE: no DB. The cross-run seen-ledger and the iter_posts +orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate +(`is_seen`), so this module needs no DB and is unit-testable without network. + +On-disk layout (matches gallery-dl): + //patreon//_ + where = "__" (date prefix omitted when the + post's published_at is unparseable), is the 1-based index of the item in + the post zero-padded to 2, and is MediaItem.filename. The sidecar + is written next to the media as _.json (media_path.with_suffix). + +Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path +endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same +Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py +_get_default_config). yt-dlp may remux to a container of its own choosing, so we +accept the actual output extension and record the real path. + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import subprocess +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from urllib.parse import urlsplit + +import requests + +from .file_validator import is_validatable, quarantine_file, validate_file +from .patreon_client import ( + _BACKOFF_CAP_SECONDS, + _load_session, + _retry_after_seconds, +) + +log = logging.getLogger(__name__) + +_TITLE_MAX = 40 +_TIMEOUT_SECONDS = 120.0 +_CHUNK = 1 << 16 +# Retry a media GET that hits a TRANSIENT failure within the same pass (plan +# #705 #8): a transport blip (connection reset / timeout / truncated stream), a +# 429, or a 5xx. PERMANENT failures (404 gone, 403 forbidden) fail fast straight +# to the error/dead-letter path — no point re-fetching them. Keeps a momentary +# network hiccup from becoming a per-item error that waits for the next walk. +_MAX_MEDIA_RETRIES = 3 +# requests transport errors worth retrying (vs. an HTTPError, which is a real +# server response and is classified by status code). +_TRANSIENT_TRANSPORT_EXC = ( + requests.ConnectionError, + requests.Timeout, + requests.exceptions.ChunkedEncodingError, +) + +# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT +# playback policy checks Referer/Origin on every request, so yt-dlp must send +# Patreon's, not its own default. (gallery-dl forwarded the same headers before +# the #697 cutover removed its Patreon path; this is now the only place they +# live.) +_VIDEO_HEADERS = { + "Referer": "https://www.patreon.com/", + "Origin": "https://www.patreon.com", +} + +# Characters Windows/gallery-dl path-restrict forbids, plus path separators. +_FORBIDDEN = set('<>:"/\\|?*') + + +def _sanitize(name: str) -> str: + """Make `name` safe for a single filesystem path segment. + + Replaces path separators, the Windows-forbidden set <>:"/\\|?* and control + characters with `_`, then strips trailing dots/spaces (gallery-dl + path-restrict behavior). Never returns empty (falls back to "_"). + """ + out = [] + for ch in name: + if ch in _FORBIDDEN or ord(ch) < 32: + out.append("_") + else: + out.append(ch) + cleaned = "".join(out).rstrip(". ") + return cleaned or "_" + + +def _is_video_url(url: str) -> bool: + parts = urlsplit(url) + if parts.hostname and parts.hostname.lower() == "stream.mux.com": + return True + return parts.path.lower().endswith(".m3u8") + + +def _post_dir_name(post: dict) -> str: + """Build the post directory name matching gallery-dl's layout.""" + post_id = str(post.get("id") or "") + attrs = post.get("attributes") or {} + title = attrs.get("title") + title = title if isinstance(title, str) else "" + title40 = title[:_TITLE_MAX] + + published = attrs.get("published_at") + date_prefix = None + if isinstance(published, str) and published: + s = published.strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(s) + except ValueError: + dt = None + if dt is not None: + date_prefix = f"{dt:%Y-%m-%d}" + + if date_prefix: + raw = f"{date_prefix}_{post_id}_{title40}" + else: + raw = f"{post_id}_{title40}" + return _sanitize(raw) + + +@dataclass +class MediaOutcome: + """Per-media result of a download_post pass. + + status is one of: "downloaded", "skipped_seen", "skipped_disk", + "quarantined", "error". `path` is the final on-disk path for "downloaded" + (the actual yt-dlp output for video), the path that already existed for + "skipped_disk", or the _quarantine destination for "quarantined"; None for + "skipped_seen" and (usually) "error". `error` carries the failure/validation + reason for "error"/"quarantined", else None. + """ + + media: object # MediaItem (avoid importing the name for a bare annotation) + status: str + path: Path | None + error: str | None + + +class PatreonDownloader: + """Download resolved Patreon media to gallery-dl's on-disk layout. + + PURE: no DB. The HTTP session and the yt-dlp invocation are injectable seams + so tests run without network or a real subprocess: + - pass `session=` to stub `session.get`, or monkeypatch `_fetch_to_file`. + - monkeypatch `_run_ytdlp` to avoid spawning yt-dlp. + """ + + def __init__( + self, + images_root: Path, + cookies_path: str | None = None, + *, + validate: bool = True, + rate_limit: float = 0.0, + session: requests.Session | None = None, + ): + self.images_root = Path(images_root) + self.cookies_path = str(cookies_path) if cookies_path else None + self._validate = validate + # Politeness: seconds to sleep before each actual media download (paces + # the CDN; honors ImportSettings.download_rate_limit_seconds, the same + # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing. + # Applied only to real downloads, not to seen/disk skips. plan #703. + self._rate_limit = rate_limit or 0.0 + # Build a cookie-loaded session the same way patreon_client does, so the + # CDN GETs carry the creator's auth. + self.session = session if session is not None else _load_session(cookies_path) + + # -- public ------------------------------------------------------------ + + def download_post( + self, + post: dict, + media_items: list, + artist_slug: str, + *, + is_seen: Callable[[object], bool] = lambda m: False, + ) -> list[MediaOutcome]: + """Download every media item of one post; return per-item outcomes. + + Builds the post directory, iterates media (1-based NN), applies the + two-tier skip (injected is_seen, then disk), downloads (plain GET or + yt-dlp for video), writes the sidecar for each freshly-downloaded item, + validates, and returns outcomes. Resilient: one media's failure yields + an "error" outcome for that item; the rest proceed. + """ + post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) + outcomes: list[MediaOutcome] = [] + + for i, media in enumerate(media_items, start=1): + try: + outcomes.append( + self._download_one(post, media, post_dir, artist_slug, i, is_seen) + ) + except Exception as exc: # resilient: isolate one item's failure + log.warning( + "Patreon media failed (post %s, item %d): %s", + post.get("id"), i, exc, + ) + outcomes.append( + MediaOutcome(media=media, status="error", path=None, error=str(exc)) + ) + return outcomes + + # -- per-item ---------------------------------------------------------- + + def _download_one( + self, + post: dict, + media, + post_dir: Path, + artist_slug: str, + index: int, + is_seen: Callable[[object], bool], + ) -> MediaOutcome: + # tier-1: seen ledger (injected; no DB here). + if is_seen(media): + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + + nn = f"{index:02d}" + final_name = _sanitize(f"{nn}_{media.filename}") + media_path = post_dir / final_name + + # tier-2: already on disk. + if media_path.exists(): + return MediaOutcome( + media=media, status="skipped_disk", path=media_path, error=None + ) + # Video may land at a different extension; honor a pre-existing remux. + if _is_video_url(media.url): + existing = self._existing_video_output(media_path) + if existing is not None: + return MediaOutcome( + media=media, status="skipped_disk", path=existing, error=None + ) + + post_dir.mkdir(parents=True, exist_ok=True) + + # Pace real downloads only (the skips above already returned). plan #703. + if self._rate_limit > 0: + time.sleep(self._rate_limit) + + if _is_video_url(media.url): + out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS) + if out_path is None or not Path(out_path).exists(): + return MediaOutcome( + media=media, + status="error", + path=None, + error="yt-dlp produced no output", + ) + out_path = Path(out_path) + else: + out_path = self._fetch_get(media.url, media_path) + reason, quarantine_dest = self._validate_path( + out_path, artist_slug, media.url + ) + if reason is not None: + # Quarantined (corrupt/invalid) — distinct from a download error + # so the run can report a real files_quarantined count + paths. + return MediaOutcome( + media=media, status="quarantined", + path=quarantine_dest, error=reason, + ) + + self._write_sidecar(post, out_path) + return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) + + # -- download seams ---------------------------------------------------- + + def _fetch_get(self, url: str, dest: Path) -> Path: + """Stream `url` to a .part file then atomic-rename to `dest`. + + Thin wrapper over `_fetch_to_file` so tests can stub either the whole + GET path (`_fetch_to_file`) or just `session.get`. + """ + part = dest.with_name(dest.name + ".part") + try: + self._fetch_to_file(url, part) + except Exception: + with contextlib.suppress(OSError): + part.unlink() + raise + os.replace(part, dest) + return dest + + def _fetch_to_file(self, url: str, dest: Path) -> None: + """Stream a non-video URL to `dest` via the (stubbable) session, retrying + TRANSIENT failures within the same pass (plan #705 #8) and RESUMING from + the bytes already on disk via a Range request when a retry follows a + mid-download cut (plan #708 B5). + + Retried (backoff): transport blips (connection reset / timeout / + truncated stream — incl. mid-download), HTTP 429 (honoring Retry-After), + and 5xx. Failed fast (no retry → HTTPError → per-item error → dead-letter + path): 4xx other than 429 (404 gone, 403 forbidden) — re-fetching a + permanent failure is pointless. + + Resume: on a retry, if bytes already landed in `dest`, ask for the rest + with `Range: bytes=-`. A 206 means the server honored it → append; a + 200 means it ignored it (served the whole file) → start clean. The caller + (_fetch_get) stages into a `.part`, so a non-range server never corrupts + the output — the worst case is re-downloading from zero, as before. + """ + attempt = 0 + while True: + have = dest.stat().st_size if dest.exists() else 0 + headers = {"Range": f"bytes={have}-"} if have > 0 else None + try: + resp = self.session.get( + url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers, + ) + if (resp.status_code == 429 or resp.status_code >= 500) \ + and attempt < _MAX_MEDIA_RETRIES: + attempt += 1 + delay = _retry_after_seconds(resp, attempt) + log.warning( + "Patreon media transient HTTP %d (%s) — backing off " + "%.1fs (retry %d/%d)", + resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES, + ) + time.sleep(delay) + continue + # A Range that starts at/past EOF (we already have the whole file) + # comes back 416 — the bytes we kept ARE the file. + if have > 0 and resp.status_code == 416: + return + # 2xx → ok; 4xx-non-429 (or an exhausted 429/5xx) → HTTPError + # (permanent for this pass) → not caught below → per-item error. + resp.raise_for_status() + # 206 → server honored the Range; append after the kept bytes. + # Anything else (200) → it served the whole file → start clean. + mode = "ab" if (have > 0 and resp.status_code == 206) else "wb" + with open(dest, mode) as fh: + for chunk in resp.iter_content(chunk_size=_CHUNK): + if chunk: + fh.write(chunk) + return + except _TRANSIENT_TRANSPORT_EXC as exc: + if attempt >= _MAX_MEDIA_RETRIES: + raise # exhausted → terminal error outcome + attempt += 1 + delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) + log.warning( + "Patreon media transport error (%s) — backing off %.1fs " + "(retry %d/%d): %s", + url, delay, attempt, _MAX_MEDIA_RETRIES, exc, + ) + time.sleep(delay) + + def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None: + """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`. + + Returns the actual output path (yt-dlp may remux to a different + container, so we resolve the real file afterward). Overridden/ + monkeypatched in tests to avoid spawning a real subprocess. + + The output template uses `dest` without its extension; yt-dlp appends + the chosen container extension. We pass Referer/Origin (Mux JWT policy) + and the cookies file. + + Mirrors the GET path's transient/permanent split (plan #705 #8): a hung + fetch (TimeoutExpired) or a spawn failure (OSError) is TRANSIENT — back + off and retry. A non-zero yt-dlp exit (CalledProcessError) is treated as + PERMANENT for this pass — yt-dlp already does its OWN internal network + retries, so a non-zero exit is effectively a real failure (private/gone/ + geo-blocked), like a 4xx on the GET path: fail fast to the per-item error + → dead-letter path. + """ + dest = Path(dest) + out_template = str(dest.with_suffix("")) + ".%(ext)s" + cmd = ["yt-dlp", "--no-progress", "-o", out_template] + for key, value in headers.items(): + cmd += ["--add-header", f"{key}:{value}"] + if self.cookies_path and os.path.isfile(self.cookies_path): + cmd += ["--cookies", self.cookies_path] + cmd.append(url) + attempt = 0 + while True: + try: + subprocess.run( + cmd, + check=True, + capture_output=True, + text=True, + timeout=_TIMEOUT_SECONDS, + ) + break + except subprocess.CalledProcessError as exc: + # Permanent for this pass — fail fast (no retry). + log.warning( + "yt-dlp failed (exit %s) for %s: %s", + exc.returncode, url, (exc.stderr or "").strip() or exc, + ) + return None + except (OSError, subprocess.TimeoutExpired) as exc: + # Transient — back off and retry, like a transport blip on a GET. + if attempt >= _MAX_MEDIA_RETRIES: + log.warning( + "yt-dlp transient failure exhausted for %s: %s", url, exc + ) + return None + attempt += 1 + delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) + log.warning( + "yt-dlp transient failure (%s) — backing off %.1fs " + "(retry %d/%d): %s", + url, delay, attempt, _MAX_MEDIA_RETRIES, exc, + ) + time.sleep(delay) + return self._existing_video_output(dest) + + def _existing_video_output(self, dest: Path) -> Path | None: + """Find a yt-dlp output for `dest` regardless of chosen extension. + + Returns `dest` itself if present, else any sibling sharing the same + stem (the remuxed container). None if nothing matched. + """ + dest = Path(dest) + if dest.exists(): + return dest + stem = dest.with_suffix("").name + parent = dest.parent + if not parent.is_dir(): + return None + for cand in sorted(parent.iterdir()): + if cand.is_file() and cand.stem == stem and cand.name != dest.name: + return cand + return None + + # -- validation -------------------------------------------------------- + + def _validate_path( + self, path: Path, artist_slug: str, source_url: str | None = None + ) -> tuple[str | None, Path | None]: + """Validate a freshly-written file; quarantine if bad. + + Uses the shared `file_validator.quarantine_file` — same move + provenance + sidecar gallery-dl writes (the native path used to skip the sidecar; that + parity gap is closed here). Returns `(reason, quarantine_dest)` when + quarantined (dest is the original path if the move itself failed), else + `(None, None)` (ok / not validatable / disabled). plan #704: the dest is + surfaced so the run reports a real quarantined-paths list. + """ + if not self._validate or not is_validatable(path): + return None, None + try: + result = validate_file(path) + except Exception as exc: + log.warning("Validator raised on %s: %s", path, exc) + return None, None + if result.ok: + return None, None + dest = quarantine_file( + self.images_root, path, artist_slug, "patreon", + url=source_url, result=result, + ) + return (result.reason or "validation failed"), (dest or path) + + # -- sidecar ----------------------------------------------------------- + + def _write_sidecar(self, post: dict, media_path: Path) -> Path: + """Write the importer-consumed sidecar next to `media_path`. + + Patreon uses base-default sidecar keys (parse_sidecar maps + category->platform, id->external_post_id, title->post_title, + content->description, published_at->post_date, url->post_url). Patreon + registers no derive_post_url, so `url` is trusted as the permalink — we + pass the post's attributes.url. + """ + attrs = post.get("attributes") or {} + title = attrs.get("title") + content = attrs.get("content") + published = attrs.get("published_at") + url = attrs.get("url") + data = { + "category": "patreon", + "id": str(post.get("id") or ""), + "title": title if isinstance(title, str) else "", + "content": content if isinstance(content, str) else "", + "published_at": published if isinstance(published, str) else None, + "url": url if isinstance(url, str) else None, + } + sidecar_path = media_path.with_suffix(".json") + sidecar_path.write_text(json.dumps(data, indent=2)) + return sidecar_path diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py new file mode 100644 index 0000000..5915da7 --- /dev/null +++ b/backend/app/services/patreon_ingester.py @@ -0,0 +1,195 @@ +"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core. + +The orchestration that drives a native subscription walk (page a feed → extract +media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor +→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery) +now lives in `ingest_core.Ingester` — it's identical for every platform. This +module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/ +ledger models/constraints/key into the core and supplies the Patreon-specific +failure mapping. `download_service.download_source` calls `PatreonIngester.run` +exactly as before; the public surface (this class, `_ledger_key`, +`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged. + +Three modes (selected by `download_service` from `config_overrides` state): + - tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out + after N contiguous already-have-it items (the cheap native + equivalent of gallery-dl's `exit:20`, now free of per-file HEADs). + - backfill — full-history walk in a time-boxed chunk, resuming from the + pagination cursor checkpoint; reaches the bottom → "complete". + - recovery — like backfill but BYPASSES the tier-1 seen-ledger AND the + dead-letter ledger, so deliberately-dropped-and-deleted near-dups + get re-fetched and re-evaluated under the current pHash threshold + (tier-2 disk skip still spares files we kept). + +The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` / +`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch +— never held across a network fetch ([[db-connection-held-across-subprocess]]). +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable +from pathlib import Path + +from ..models import PatreonFailedMedia, PatreonSeenMedia +from .gallery_dl import DownloadResult, ErrorType +from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester +from .patreon_client import ( + MediaItem, + PatreonAPIError, + PatreonAuthError, + PatreonClient, + PatreonDriftError, +) +from .patreon_downloader import PatreonDownloader +from .patreon_resolver import resolve_campaign_id_for_source + +__all__ = [ + "DEAD_LETTER_THRESHOLD", + "PatreonIngester", + "_ledger_key", + "verify_patreon_credential", +] + +log = logging.getLogger(__name__) + +# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any +# synthesized key so a pathologically long file_name can't overflow the column. +_LEDGER_KEY_MAX = 128 + + +def _ledger_key(media: MediaItem) -> str: + """Stable per-media identity for the cross-run seen-ledger. + + A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the + natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no + content hash at discovery) and the odd inline-content `` pointing at a + hashless URL. The plan calls the video case the ``video::`` + sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the + stable proxy. Bounded to the column width. + """ + if media.filehash: + return media.filehash + return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX] + + +class PatreonIngester(Ingester): + """Walk a Patreon campaign's posts, download unseen media, return a + `DownloadResult`. A thin adapter over `ingest_core.Ingester`. + + Construct with the per-source `cookies_path` and a sync sessionmaker for the + ledgers. `client` / `downloader` are injectable seams so unit tests run + without network, subprocess, or a real CDN. + """ + + def __init__( + self, + images_root: Path, + cookies_path: str | None, + session_factory: Callable[[], object], + *, + validate: bool = True, + rate_limit: float = 0.0, + request_sleep: float = 0.0, + client: PatreonClient | None = None, + downloader: PatreonDownloader | None = None, + ): + self.images_root = Path(images_root) + self.cookies_path = str(cookies_path) if cookies_path else None + # Pacing (plan #703): request_sleep paces the API page fetches, + # rate_limit paces the media downloads. Injected client/downloader (in + # tests) already carry their own pacing, so these only apply to the + # default-constructed ones. + resolved_client = ( + client + if client is not None + else PatreonClient(cookies_path, request_sleep=request_sleep) + ) + resolved_downloader = ( + downloader + if downloader is not None + else PatreonDownloader( + self.images_root, cookies_path, validate=validate, rate_limit=rate_limit + ) + ) + super().__init__( + client=resolved_client, + downloader=resolved_downloader, + session_factory=session_factory, + seen_model=PatreonSeenMedia, + failed_model=PatreonFailedMedia, + seen_constraint="uq_patreon_seen_media_source_id", + failed_constraint="uq_patreon_failed_media_source_id", + ledger_key=_ledger_key, + platform="patreon", + error_base=PatreonAPIError, + ) + + # -- failure mapping (Patreon exception taxonomy) ---------------------- + + def _failure_result(self, exc: Exception, _result) -> DownloadResult: + """Map a client-level exception to a loud, typed failed DownloadResult. + + We NEVER return a silent zero-download "success" — the whole point of the + native ingester is to fail RED when Patreon's API shape or our auth + changes. The typed mapping lets FailingSourcesCard render the right chip + and tells the operator what to do: + - PatreonAuthError → AUTH_ERROR (rotate cookies) + - PatreonDriftError → API_DRIFT (ingester field-set/parser needs update) + - HTTP 429 / 404 → RATE_LIMITED / NOT_FOUND + - other HTTP status → HTTP_ERROR; transport failure → NETWORK_ERROR + + PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so + they must be matched before the generic HTTP/transport fallthrough. + """ + message = str(exc) + if isinstance(exc, PatreonAuthError): + error_type = ErrorType.AUTH_ERROR + elif isinstance(exc, PatreonDriftError): + error_type = ErrorType.API_DRIFT + message = f"Patreon API changed — ingester needs update: {message}" + else: # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport + status = getattr(exc, "status_code", None) + if status == 429: + error_type = ErrorType.RATE_LIMITED + elif status == 404: + error_type = ErrorType.NOT_FOUND + elif status is not None: + error_type = ErrorType.HTTP_ERROR + else: + error_type = ErrorType.NETWORK_ERROR + log.warning("Patreon ingest failed (%s): %s", error_type.value, message) + result = _result( + success=False, return_code=1, + error_type=error_type, error_message=message, + ) + # plan #708 B1: carry the server's Retry-After up to the cooldown. + if error_type == ErrorType.RATE_LIMITED: + result.retry_after_seconds = getattr(exc, "retry_after", None) + return result + + +async def verify_patreon_credential( + url: str, + cookies_path: str | None, + overrides: dict | None, +) -> tuple[bool | None, str]: + """Native Patreon credential probe — the verify counterpart to the ingester's + download path, sharing its campaign-id resolution. Resolves the campaign id + (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch + via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract + (True / False / None) so download_backends.verify_credential can treat it + interchangeably with the gallery-dl probe. No download, no DB. + """ + campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides) + if not campaign_id: + return None, ( + "Couldn't resolve the Patreon campaign id from the source URL — " + "can't verify (cookies expired, or the creator moved/renamed?)." + ) + client = PatreonClient(cookies_path) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, client.verify_auth, campaign_id) diff --git a/backend/app/services/patreon_resolver.py b/backend/app/services/patreon_resolver.py index 64797c9..fb29661 100644 --- a/backend/app/services/patreon_resolver.py +++ b/backend/app/services/patreon_resolver.py @@ -19,12 +19,22 @@ import asyncio import http.cookiejar import logging import os +import re import requests log = logging.getLogger(__name__) _CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns" + +# A source URL of the form `.../id:` already carries the campaign id +# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the +# two paths don't overlap. +_ID_URL_RE = re.compile(r"/id:(\d+)") +_VANITY_RE = re.compile( + r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", + re.IGNORECASE, +) _USER_AGENT = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" @@ -100,3 +110,38 @@ async def resolve_campaign_id( Never raises.""" loop = asyncio.get_running_loop() return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path) + + +def extract_vanity(url: str) -> str | None: + """The vanity slug from a Patreon creator URL, or None for an `id:` URL.""" + m = _VANITY_RE.match(url or "") + return m.group(1) if m else None + + +async def resolve_campaign_id_for_source( + url: str, + cookies_path: str | None, + overrides: dict | None, +) -> tuple[str | None, str | None]: + """Resolve a Patreon source to its campaign id — the single resolution path + shared by the download ingester and the credential-verify probe. + + Order: cached `patreon_campaign_id` override → an `id:` URL → a + vanity lookup against the campaigns API. Returns + `(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when + a vanity lookup actually ran, so the caller knows to cache it on the source + (the override/id: paths needed no lookup). `(None, None)` when unresolvable. + Never raises. + """ + overrides = overrides or {} + cached = overrides.get("patreon_campaign_id") + if cached: + return cached, None + id_match = _ID_URL_RE.search(url or "") + if id_match: + return id_match.group(1), None + vanity = extract_vanity(url) + if vanity: + resolved = await resolve_campaign_id(vanity, cookies_path) + return resolved, resolved + return None, None diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 7ae8f9c..073b4d8 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -67,6 +67,12 @@ class SourceRecord: # plan #693: derived from config_overrides for the UI badge. backfill_state: str | None # "running" | "complete" | "stalled" | None (idle) backfill_chunks: int + # plan #697: a running deep-walk that bypasses the Patreon seen-ledger + # (recovery) vs. a normal backfill. Lets the badge label it "Recovering". + backfill_bypass_seen: bool + # plan #704: cumulative posts processed across the walk's chunks — live + # progress for the badge. + backfill_posts: int def to_dict(self) -> dict: return { @@ -87,6 +93,8 @@ class SourceRecord: "backfill_runs_remaining": self.backfill_runs_remaining, "backfill_state": self.backfill_state, "backfill_chunks": self.backfill_chunks, + "backfill_bypass_seen": self.backfill_bypass_seen, + "backfill_posts": self.backfill_posts, } @@ -162,6 +170,8 @@ class SourceService: backfill_runs_remaining=source.backfill_runs_remaining or 0, backfill_state=co.get("_backfill_state"), backfill_chunks=int(co.get("_backfill_chunks", 0)), + backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), + backfill_posts=int(co.get("_backfill_posts", 0)), ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -312,7 +322,37 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) co["_backfill_state"] = "running" - for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"): + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): + co.pop(k, None) + source.config_overrides = co + source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + await self.session.commit() + return await self._row_to_record(source) + + async def start_recovery(self, source_id: int) -> SourceRecord: + """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon + seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched + and re-evaluated under the CURRENT pHash threshold (tier-2 disk still + spares files we kept). Reuses the entire #693 backfill state machine + (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the + `_backfill_bypass_seen` flag that flips download mode to recovery. Clears + any prior cursor/chunk/stall state so it walks fresh from the top. The + flag is cleared on completion (download_service) and on stop. + + Recovery is Patreon-only (the seen-ledger is Patreon's); for other + platforms the flag is inert (download_service ignores it) and the walk + runs as a plain backfill. The UI gates the action to Patreon sources.""" + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + co = dict(source.config_overrides or {}) + co["_backfill_state"] = "running" + co["_backfill_bypass_seen"] = True + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS @@ -329,7 +369,7 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls", - "_backfill_chunks"): + "_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = 0 diff --git a/backend/app/tasks/download.py b/backend/app/tasks/download.py index d72e78b..42c075d 100644 --- a/backend/app/tasks/download.py +++ b/backend/app/tasks/download.py @@ -143,6 +143,11 @@ def download_source(self, source_id: int) -> int: gdl=gdl, importer=importer, cred_service=cred_service, + # The native Patreon ingester opens its own short-lived + # sync sessions for the seen-ledger (never held across + # the walk). Same factory the importer's sync session + # comes from — a different DB connection per checkout. + sync_session_factory=SyncFactory, ) return await svc.download_source(source_id) finally: diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py index c19b7f8..d9bc48e 100644 --- a/backend/app/utils/paths.py +++ b/backend/app/utils/paths.py @@ -2,6 +2,26 @@ from pathlib import Path +_MAX_EXT_LEN = 16 + + +def safe_ext(name: str | Path) -> str: + """Conservatively extract a short, alphanumeric file extension. + + gallery-dl and Patreon CDN URLs produce basenames with URL-encoded + query-string artifacts, so `Path.suffix` can return 50+ chars of base64-ish + junk that blows bounded VARCHAR columns (e.g. PostAttachment.ext varchar(32)). + Accept only a suffix ≤16 chars whose post-dot characters are all alphanumeric; + otherwise return "" (no known extension). Operator-flagged 2026-05-25 — ONE + impl for the importer and the native Patreon client. + """ + suffix = Path(name).suffix.lower() + if not suffix or len(suffix) > _MAX_EXT_LEN: + return "" + if not all(c.isalnum() for c in suffix[1:]): + return "" + return suffix + def derive_subdir(source_path: Path, import_root: Path) -> str: """Returns the relative subdirectory of source_path under import_root. diff --git a/frontend/src/components/subscriptions/FailingSourcesCard.vue b/frontend/src/components/subscriptions/FailingSourcesCard.vue index 024a4df..f456904 100644 --- a/frontend/src/components/subscriptions/FailingSourcesCard.vue +++ b/frontend/src/components/subscriptions/FailingSourcesCard.vue @@ -93,6 +93,7 @@ const ERROR_TYPE_COLOR = { unsupported_url: 'error', http_error: 'error', unknown_error: 'error', + api_drift: 'error', partial: 'info', tier_limited: 'info', no_new_content: 'info', @@ -108,6 +109,7 @@ const ERROR_TYPE_HINT = { http_error: 'Generic HTTP error — see Logs.', unsupported_url: 'gallery-dl does not support this URL pattern.', unknown_error: 'Could not classify — see Logs.', + api_drift: 'Patreon changed its API shape — the native ingester needs a code update.', } function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' } function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' } diff --git a/frontend/src/components/subscriptions/PreviewDialog.vue b/frontend/src/components/subscriptions/PreviewDialog.vue new file mode 100644 index 0000000..7639526 --- /dev/null +++ b/frontend/src/components/subscriptions/PreviewDialog.vue @@ -0,0 +1,113 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index 7b8f9bc..0e0617b 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -27,7 +27,10 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_posts + ? ` · ${source.backfill_posts} posts` + : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} - {{ source.backfill_state === 'running' ? 'Stop backfill' : 'Backfill full history' }} + {{ source.backfill_state === 'running' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') + : 'Backfill full history' }} + + + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold @@ -83,7 +108,7 @@ const props = defineProps({ checking: { type: Boolean, default: false }, warningThreshold: { type: Number, default: 5 }, }) -const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill']) +const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'preview']) function onToggleEnabled(value) { emit('toggle', { source: props.source, enabled: value }) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 9fe412e..22873be 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -34,7 +34,10 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_posts + ? ` · ${source.backfill_posts} posts` + : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} {{ source.backfill_state === 'running' - ? 'Stop backfill' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') : 'Backfill — walk the full post history until complete' }} + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold + + @@ -251,6 +253,8 @@ @toggle="toggleSourceEnabled" @check="onCheck" @backfill="onBackfill" + @recover="onRecover" + @preview="onPreview" />
No sources yet. Tap + to add one. @@ -266,6 +270,11 @@ @saved="onSourceSaved" /> +
@@ -282,6 +291,7 @@ import SourceCard from './SourceCard.vue' import SourceHealthDot from './SourceHealthDot.vue' import SourceFormDialog from './SourceFormDialog.vue' import ArtistCreateDialog from './ArtistCreateDialog.vue' +import PreviewDialog from './PreviewDialog.vue' import PlatformChip from './PlatformChip.vue' import SchedulerStatusBar from './SchedulerStatusBar.vue' import { formatRelative } from '../../utils/date.js' @@ -332,6 +342,8 @@ const showSourceDialog = ref(false) const editingSource = ref(null) const editingArtist = ref(null) const showArtistDialog = ref(false) +const showPreviewDialog = ref(false) +const previewTarget = ref(null) const artistFilter = computed(() => { const raw = route.query.artist_id @@ -548,12 +560,40 @@ async function onBackfill(source) { await store.loadAll() } catch (e) { toast({ - text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.detail || e?.message || e}`, + text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.body?.detail || e?.message || e}`, type: 'error', }) } } +// Plan #697: arm a recovery walk (Patreon-only) — re-fetches dropped-and-deleted +// near-dups and re-evaluates them under the current pHash threshold. Reuses the +// backfill lifecycle/badge; stop via the same Stop control (onBackfill). +async function onRecover(source) { + try { + await store.recoverSource(source.id, source.artist_id) + toast({ text: `Recovery started for ${source.artist_name}`, type: 'success' }) + await store.loadAll() + } catch (e) { + toast({ + text: `Recovery start failed: ${e?.body?.detail || e?.message || e}`, + type: 'error', + }) + } +} + +// Plan #708 B4: open the dry-run preview dialog (it self-fetches on open). +function onPreview(source) { + previewTarget.value = source + showPreviewDialog.value = true +} + +// "Start backfill" from inside the preview dialog → arm it + close. +async function onPreviewBackfill(source) { + showPreviewDialog.value = false + await onBackfill(source) +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index 5eb90fe..f16ad8e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -98,6 +98,20 @@ export const useSourcesStore = defineStore('sources', () => { _invalidate(artistIdHint ?? body.artist_id) return body } + // Plan #697: arm a recovery walk — a backfill that bypasses the Patreon + // seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate them + // under the current pHash threshold. Stop with stopBackfill (shared lifecycle). + async function recoverSource(id, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recover' } }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } + + // Plan #708 B4: dry-run preview — count what a backfill WOULD download + // (native platforms), without downloading. Read-only, so no cache invalidate. + async function previewSource(id) { + return await api.post(`/api/sources/${id}/preview`) + } function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] @@ -127,6 +141,8 @@ export const useSourcesStore = defineStore('sources', () => { checkNow, startBackfill, stopBackfill, + recoverSource, + previewSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/fixtures/patreon_posts_page1.json b/tests/fixtures/patreon_posts_page1.json new file mode 100644 index 0000000..6515c34 --- /dev/null +++ b/tests/fixtures/patreon_posts_page1.json @@ -0,0 +1,121 @@ +{ + "data": [ + { + "id": "1001", + "type": "post", + "attributes": { + "title": "Image gallery post", + "published_at": "2026-05-01T12:00:00.000+00:00", + "post_type": "image_file", + "content": "

just a gallery

", + "url": "https://www.patreon.com/posts/1001", + "image": { + "large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + }, + "relationships": { + "images": { + "data": [ + {"id": "9001", "type": "media"}, + {"id": "9002", "type": "media"} + ] + } + } + }, + { + "id": "1002", + "type": "post", + "attributes": { + "title": "Attachment post", + "published_at": "2026-04-15T09:30:00.000+00:00", + "post_type": "text_only", + "content": "

here is a zip

", + "url": "https://www.patreon.com/posts/1002", + "post_file": { + "name": "bonus-pack.zip", + "url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002" + } + }, + "relationships": { + "attachments_media": { + "data": [ + {"id": "9003", "type": "media"} + ] + } + } + }, + { + "id": "1003", + "type": "post", + "attributes": { + "title": "Inline content post", + "published_at": "2026-04-01T08:00:00.000+00:00", + "post_type": "text_only", + "content": "

look

\"inline\"
", + "url": "https://www.patreon.com/posts/1003" + }, + "relationships": {} + }, + { + "id": "1004", + "type": "post", + "attributes": { + "title": "Video post", + "published_at": "2026-03-20T18:45:00.000+00:00", + "post_type": "video_external_file", + "content": "

watch

", + "url": "https://www.patreon.com/posts/1004", + "post_file": { + "name": "clip.m3u8", + "url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt" + } + }, + "relationships": {} + } + ], + "included": [ + { + "id": "9001", + "type": "media", + "attributes": { + "file_name": "gallery-one.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + } + }, + { + "id": "9002", + "type": "media", + "attributes": { + "file_name": "gallery-two.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg" + } + } + }, + { + "id": "9003", + "type": "media", + "attributes": { + "file_name": "bonus-pack.zip", + "download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc" + } + }, + { + "id": "5555", + "type": "campaign", + "attributes": { + "name": "Example Creator", + "url": "https://www.patreon.com/example" + } + } + ], + "links": { + "next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at" + } +} diff --git a/tests/test_api_credentials.py b/tests/test_api_credentials.py index fa042b4..43a4853 100644 --- a/tests/test_api_credentials.py +++ b/tests/test_api_credentials.py @@ -155,6 +155,8 @@ async def test_verify_no_enabled_source_is_untestable(client): @pytest.mark.asyncio async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch): + """A gallery-dl platform routes through GalleryDLService.verify; success + stamps last_verified (platform-agnostic endpoint behavior).""" from backend.app.models import Artist, Source from backend.app.services import gallery_dl as gdl_mod @@ -163,6 +165,45 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa return (True, "Credentials valid — the feed authenticated.") monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify) + await client.post("/api/credentials", json={ + "platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE, + }) + artist = Artist(name="Maewix", slug="maewix") + db.add(artist) + await db.flush() + db.add(Source( + artist_id=artist.id, platform="subscribestar", + url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={}, + )) + await db.commit() + + resp = await client.post("/api/credentials/subscribestar/verify") + body = await resp.get_json() + assert body["valid"] is True + assert body["last_verified"] is not None + + # The stamp is persisted on the credential record. + rec = await (await client.get("/api/credentials/subscribestar")).get_json() + assert rec["last_verified"] is not None + + +@pytest.mark.asyncio +async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch): + """plan #697 cutover: Patreon credential verify routes through the native + ingester (verify_patreon_credential), NOT gallery-dl --simulate.""" + from backend.app.models import Artist, Source + from backend.app.services import gallery_dl as gdl_mod + from backend.app.services import patreon_ingester as pi_mod + + async def _native_verify(url, cookies_path, overrides): + return (True, "Credentials valid — the Patreon feed authenticated.") + monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify) + + # gallery-dl must NOT be consulted for Patreon. + async def _boom(self, *args, **kwargs): + raise AssertionError("gallery-dl verify must not run for patreon") + monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom) + await client.post("/api/credentials", json={ "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE, }) @@ -180,10 +221,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa assert body["valid"] is True assert body["last_verified"] is not None - # The stamp is persisted on the credential record. - rec = await (await client.get("/api/credentials/patreon")).get_json() - assert rec["last_verified"] is not None - @pytest.mark.asyncio async def test_verify_reports_auth_failure(client, db, monkeypatch): diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 3a13d97..71f4dde 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -5,6 +5,20 @@ from backend.app.models import Artist, Source pytestmark = pytest.mark.integration +@pytest.fixture(autouse=True) +def _stub_preflight_verify(monkeypatch): + """Backfill/recover arms run a pre-flight credential verify (plan #703 #2) + which for Patreon would hit the network. Default it to 'valid' so the + endpoint tests stay network-free; the dedicated pre-flight tests override + this with a rejection/inconclusive verdict.""" + from backend.app.services import download_backends as db_mod + + async def _ok(**kwargs): + return (True, "Credentials valid") + + monkeypatch.setattr(db_mod, "verify_source_credential", _ok) + + @pytest.fixture async def artist(db): a = Artist(name="Alice", slug="alice") @@ -250,3 +264,117 @@ async def test_backfill_endpoint_rejects_bad_action(client, artist, db): async def test_backfill_endpoint_404_when_source_missing(client): resp = await client.post("/api/sources/999999/backfill", json={"action": "start"}) assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_recover_arms_bypass(client, artist, db): + """Plan #697: action='recover' arms a backfill that bypasses the Patreon + seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge + can label it 'Recovering'. Stop clears it.""" + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-recover", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_state"] == "running" + assert body["backfill_bypass_seen"] is True + + stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"}) + stopped_body = await stopped.get_json() + assert stopped_body["backfill_state"] is None + assert stopped_body["backfill_bypass_seen"] is False + + +# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm ----- + + +@pytest.mark.asyncio +async def test_arm_blocked_when_credential_rejected(client, artist, db, monkeypatch): + """A definitively-rejected credential refuses the arm (409 + reason) instead + of starting a doomed walk; the source is NOT armed.""" + from backend.app.services import download_backends as db_mod + + async def _reject(**kwargs): + return (False, "Patreon rejected the credential — cookies expired") + monkeypatch.setattr(db_mod, "verify_source_credential", _reject) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-preflight", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"}) + assert resp.status_code == 409 + assert "expired" in ((await resp.get_json()).get("detail") or "") + + # Not armed. + one = await (await client.get(f"/api/sources/{sid}")).get_json() + assert one["backfill_state"] is None + + +@pytest.mark.asyncio +async def test_arm_proceeds_when_credential_inconclusive(client, artist, db, monkeypatch): + """An inconclusive verify (network blip / drift) must NOT block the arm.""" + from backend.app.services import download_backends as db_mod + + async def _inconclusive(**kwargs): + return (None, "couldn't verify (network)") + monkeypatch.setattr(db_mod, "verify_source_credential", _inconclusive) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-incon", enabled=True, + ) + db.add(src) + await db.commit() + + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"}) + assert resp.status_code == 200 + assert (await resp.get_json())["backfill_state"] == "running" + + +@pytest.mark.asyncio +async def test_stop_never_pre_flights(client, artist, db, monkeypatch): + from backend.app.services import download_backends as db_mod + + async def _boom(**kwargs): + raise AssertionError("stop must not pre-flight verify") + monkeypatch.setattr(db_mod, "verify_source_credential", _boom) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-stop", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "stop"}) + assert resp.status_code == 200 + + +@pytest.mark.asyncio +async def test_gallery_dl_platform_arm_skips_pre_flight(client, artist, db, monkeypatch): + """Pre-flight is gated to native-ingester platforms (cheap verify); a + gallery-dl source arms without the slow --simulate probe.""" + from backend.app.services import download_backends as db_mod + + async def _boom(**kwargs): + raise AssertionError("gallery-dl arm must not pre-flight verify") + monkeypatch.setattr(db_mod, "verify_source_credential", _boom) + + src = Source( + artist_id=artist.id, platform="subscribestar", + url="https://www.subscribestar.com/x", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"}) + assert resp.status_code == 200 diff --git a/tests/test_download_backends.py b/tests/test_download_backends.py new file mode 100644 index 0000000..3c67fd9 --- /dev/null +++ b/tests/test_download_backends.py @@ -0,0 +1,23 @@ +"""download_backends — the single predicate that routes a platform to the +native ingester vs. gallery-dl. Pure, no DB.""" + +from backend.app.services.download_backends import ( + NATIVE_INGESTER_PLATFORMS, + uses_native_ingester, +) + + +def test_patreon_is_native(): + assert uses_native_ingester("patreon") is True + assert "patreon" in NATIVE_INGESTER_PLATFORMS + + +def test_gallery_dl_platforms_are_not_native(): + # The five platforms still served by gallery-dl must NOT route to the + # native ingester — guards an accidental over-broad migration. + for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"): + assert uses_native_ingester(platform) is False + + +def test_unknown_platform_is_not_native(): + assert uses_native_ingester("nonsense") is False diff --git a/tests/test_download_service.py b/tests/test_download_service.py index ee5766f..a4b4779 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -59,7 +59,7 @@ def _make_jpg(path: Path, split: str = "h"): def _make_fake_dl_result( *, success=True, written_paths=None, quarantined_paths=None, files_downloaded=0, error_type=None, error_message=None, - stdout="", stderr="", + stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0, ): return SimpleNamespace( success=success, @@ -78,12 +78,22 @@ def _make_fake_dl_result( duration_seconds=1.23, started_at="2026-05-20T14:00:00+00:00", completed_at="2026-05-20T14:01:00+00:00", + # plan #704: native ingester now returns the cursor + run_stats + # structurally (no more stderr `Cursor:` scraping). + cursor=cursor, + run_stats=run_stats, + posts_processed=posts_processed, ) def _fake_gdl_with_result(result): fake = MagicMock() fake.download = AsyncMock(return_value=result) + # Real values for the attrs run_download's native branch reads off the gdl + # (the native pacing config, plan #703) — a MagicMock would break the + # arithmetic (max(0.5, rate_limit/4)). + fake._rate_limit = 3.0 + fake._validate_files = True fake._compute_run_stats = lambda *a, **k: { "exit_code": 0, "downloaded_count": len(result.written_paths), "skipped_count": 0, "per_item_failures": 0, @@ -94,6 +104,28 @@ def _fake_gdl_with_result(result): return fake +def _stub_patreon_ingester(svc, result, resolved_campaign_id=None): + """Route the phase-2 dispatch (download_backends.run_download via + DownloadService._run_download, plan #707 A5) to a canned DownloadResult so + these tests exercise download_service's phase-2-result→phase-3 handling + (status mapping, backfill lifecycle, health) without the real ingester's + network/DB. The ingester itself is covered by test_patreon_ingester.py; the + native construction/resolution by the run_download tests below. Returns a list + capturing (ctx, source_config, skip_value, mode) per call so a test can assert + mode / resume_cursor threading.""" + calls = [] + + async def fake(*, ctx, source_config, skip_value, mode): + calls.append({ + "ctx": ctx, "source_config": source_config, + "skip_value": skip_value, "mode": mode, + }) + return result, resolved_campaign_id + + svc._run_download = fake + return calls + + @pytest.mark.asyncio async def test_download_source_attaches_written_files( db, db_sync, tmp_path, seed_artist_and_source, @@ -112,12 +144,13 @@ async def test_download_source_attaches_written_files( _make_jpg(f1, split="h") _make_jpg(f2, split="v") - fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + result = _make_fake_dl_result( success=True, written_paths=[str(f1), str(f2)], files_downloaded=2, stdout=f"{f1}\n{f2}\n", - )) + ) + fake_gdl = _fake_gdl_with_result(result) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -137,6 +170,7 @@ async def test_download_source_attaches_written_files( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, result) event_id = await svc.download_source(source.id) ev = (await db.execute( @@ -214,40 +248,19 @@ async def test_in_flight_idempotency_returns_existing_event( @pytest.mark.asyncio -async def test_patreon_retry_caches_campaign_id( - db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +async def test_patreon_resolved_campaign_id_is_cached( + db, db_sync, tmp_path, seed_artist_and_source, ): + """plan #697: the native ingester resolves a Patreon vanity → campaign id + on the fly (replacing gallery-dl's reactive campaign-id retry). When phase 2 + reports a freshly-resolved id, phase 3 caches it on the source so later runs + skip the lookup. Stub the ingester to report a resolved id; assert it lands + in config_overrides.""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer _artist, source = seed_artist_and_source - failed = _make_fake_dl_result( - success=False, written_paths=[], stdout="", - stderr="[patreon][error] Failed to extract campaign ID for alice\n", - ) - succeeded = _make_fake_dl_result(success=True, written_paths=[]) - - download_calls = [] - - async def fake_download(url, **k): - download_calls.append(url) - return failed if len(download_calls) == 1 else succeeded - - fake_gdl = MagicMock() - fake_gdl.download = fake_download - fake_gdl._compute_run_stats = lambda *a, **k: { - "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, - "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, - } - fake_gdl._extract_errors_warnings = lambda *a, **k: "" - fake_gdl._truncate_log = lambda x, **k: x - - monkeypatch.setattr( - "backend.app.services.download_service.resolve_campaign_id", - AsyncMock(return_value="99"), - ) - sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() @@ -261,19 +274,97 @@ async def test_patreon_retry_caches_campaign_id( svc = DownloadService( async_session=db, sync_session=db_sync, - gdl=fake_gdl, importer=importer, cred_service=cred_service, + gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), + importer=importer, cred_service=cred_service, + ) + _stub_patreon_ingester( + svc, _make_fake_dl_result(success=True, written_paths=[]), + resolved_campaign_id="99", ) await svc.download_source(source.id) - assert len(download_calls) == 2 - assert "id:99" in download_calls[1] - overrides = db_sync.execute( select(Source.config_overrides).where(Source.id == source.id) ).scalar_one() assert overrides.get("patreon_campaign_id") == "99" +@pytest.mark.asyncio +async def test_run_download_native_resolves_vanity_and_runs( + db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +): + """run_download (native branch, plan #707 A5): with no cached campaign id, + resolve the vanity and pass the resolved id straight to the ingester; report + it back so phase 3 can cache it.""" + from backend.app.services import download_backends as db_mod + from backend.app.services.gallery_dl import SourceConfig + + _artist, source = seed_artist_and_source + + monkeypatch.setattr( + db_mod, "resolve_campaign_id_for_source", + AsyncMock(return_value=("4242", "4242")), + ) + run_kwargs = {} + + class _FakeIngester: + def __init__(self, **kw): + pass + + def run(self, **kw): + run_kwargs.update(kw) + return _make_fake_dl_result(success=True, written_paths=[]) + + monkeypatch.setattr(db_mod, "PatreonIngester", _FakeIngester) + + ctx = { + "platform": "patreon", + "source_id": source.id, "url": "https://patreon.com/alice", + "artist_slug": "alice", "cookies_path": None, + "config_overrides": {}, + } + result, resolved = await db_mod.run_download( + ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True, + mode="tick", + gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), + sync_session_factory=MagicMock(), + ) + assert result.success is True + assert resolved == "4242" + assert run_kwargs["campaign_id"] == "4242" + assert run_kwargs["mode"] == "tick" + + +@pytest.mark.asyncio +async def test_run_download_native_unresolvable_fails_loud( + db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +): + """A campaign id we can't resolve is a loud NOT_FOUND failure, never a + silent empty success.""" + from backend.app.services import download_backends as db_mod + from backend.app.services.gallery_dl import ErrorType, SourceConfig + + _artist, source = seed_artist_and_source + + monkeypatch.setattr( + db_mod, "resolve_campaign_id_for_source", + AsyncMock(return_value=(None, None)), + ) + ctx = { + "platform": "patreon", + "source_id": source.id, "url": "https://patreon.com/alice", + "artist_slug": "alice", "cookies_path": None, + "config_overrides": {}, + } + result, resolved = await db_mod.run_download( + ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True, + mode="tick", gdl=MagicMock(), sync_session_factory=MagicMock(), + ) + assert result.success is False + assert result.error_type == ErrorType.NOT_FOUND + assert resolved is None + + # --- FC-3d: finalize hook updates Source health columns ------------------- @@ -346,6 +437,40 @@ async def test_finalize_error_increments_failures_and_sets_error(db): assert row.last_checked_at is not None +@pytest.mark.asyncio +async def test_rate_limited_cooldown_honors_retry_after(db, monkeypatch): + """plan #708 B1: a RATE_LIMITED result with a Retry-After stamps the platform + cooldown at that duration, clamped to [60, 3600]; no hint → the flat default.""" + from unittest.mock import AsyncMock, MagicMock + + from backend.app.services import download_service as dl_mod + from backend.app.services.download_service import DownloadService + + source_id, _event_id = await _seed_source_with_health(db, suffix="rl") + cooldown = AsyncMock() + monkeypatch.setattr(dl_mod, "set_platform_cooldown", cooldown) + svc = DownloadService( + async_session=db, sync_session=None, + gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(), + ) + + async def _health(retry): + cooldown.reset_mock() + await svc._update_source_health( + source_id=source_id, status="error", error_message="rate limited", + error_type="rate_limited", retry_after_seconds=retry, + ) + + await _health(120.0) + assert cooldown.await_args.kwargs["seconds"] == 120 # honored as-is + await _health(5.0) + assert cooldown.await_args.kwargs["seconds"] == 60 # floored + await _health(99999.0) + assert cooldown.await_args.kwargs["seconds"] == 3600 # capped + await _health(None) + assert "seconds" not in cooldown.await_args.kwargs # default cooldown + + @pytest.mark.asyncio async def test_finalize_skipped_preserves_failures_clears_error(db): from backend.app.services.download_service import DownloadService @@ -376,8 +501,12 @@ async def test_finalize_skipped_preserves_failures_clears_error(db): def _backfill_svc(db, db_sync, tmp_path, result): - """DownloadService wired with a fake gdl returning `result` + a real - importer (so an empty written_paths just attaches nothing).""" + """DownloadService wired with a real importer (so empty written_paths just + attaches nothing) whose Patreon phase-2 branch is stubbed to return `result`. + + The seeded source is Patreon, so phase 2 routes to the native ingester + (plan #697); the stub returns the canned result + captures the per-call + (ctx, source_config, mode). Returns (svc, ingester_calls).""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer @@ -395,7 +524,8 @@ def _backfill_svc(db, db_sync, tmp_path, result): async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) - return svc, fake_gdl + calls = _stub_patreon_ingester(svc, result) + return svc, calls @pytest.mark.asyncio @@ -412,7 +542,7 @@ async def test_backfill_chunk_progress_advances_cursor( svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], files_downloaded=0, - stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n", + cursor="03:PAGE2:xyz", )) await svc.download_source(source.id) @@ -430,28 +560,25 @@ async def test_backfill_chunk_progress_advances_cursor( async def test_backfill_state_running_selects_backfill_mode_and_resumes( db, db_sync, tmp_path, seed_artist_and_source, ): - """state=='running' drives backfill mode (skip:True + chunk budget) and - threads the stored cursor to gallery-dl as the resume point.""" - from backend.app.services.gallery_dl import ( - BACKFILL_CHUNK_SECONDS, - BACKFILL_SKIP_VALUE, - ) + """state=='running' drives backfill mode (chunk budget) and threads the + stored cursor to the native ingester as the resume point.""" + from backend.app.services.gallery_dl import BACKFILL_CHUNK_SECONDS _artist, source = seed_artist_and_source source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:RESUME:here"} source.backfill_runs_remaining = 5 await db.commit() - svc, fake_gdl = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( + svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], - stderr="[patreon][debug] Cursor: 03:RESUME2:next\n", + cursor="03:RESUME2:next", )) await svc.download_source(source.id) - kwargs = fake_gdl.download.call_args.kwargs - assert kwargs["skip_value"] is BACKFILL_SKIP_VALUE - assert kwargs["source_config"].resume_cursor == "03:RESUME:here" - assert kwargs["source_config"].timeout == BACKFILL_CHUNK_SECONDS + assert len(calls) == 1 + assert calls[0]["mode"] == "backfill" + assert calls[0]["source_config"].resume_cursor == "03:RESUME:here" + assert calls[0]["source_config"].timeout == BACKFILL_CHUNK_SECONDS co = (await db.execute( select(Source.config_overrides).where(Source.id == source.id) @@ -497,7 +624,7 @@ async def test_backfill_cap_exhaustion_stalls( svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], - stderr="[patreon][debug] Cursor: 03:MORE:left\n", + cursor="03:MORE:left", )) await svc.download_source(source.id) @@ -533,7 +660,7 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance( "config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"}, "backfill_runs_remaining": 5, } - stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n") + stuck = _make_fake_dl_result(success=False, cursor="stuck") await svc._apply_backfill_lifecycle(ctx, stuck) await db.commit() @@ -553,6 +680,41 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance( assert "_backfill_cursor" not in co2 +@pytest.mark.asyncio +async def test_backfill_lifecycle_leaves_posts_to_ingester( + db, db_sync, tmp_path, seed_artist_and_source, +): + """plan #704 (#5), revised: _backfill_posts is OWNED by the ingester now (it + writes a monotonic absolute live mid-walk), so the post-chunk lifecycle must + NOT touch it — it carries the ingester's committed value forward unchanged.""" + from unittest.mock import MagicMock + + from backend.app.services.download_service import DownloadService + + _artist, source = seed_artist_and_source + source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 12} + source.backfill_runs_remaining = 5 + await db.commit() + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(), + ) + ctx = { + "source_id": source.id, "platform": "patreon", + "config_overrides": {"_backfill_state": "running", "_backfill_posts": 12}, + "backfill_runs_remaining": 5, + } + await svc._apply_backfill_lifecycle( + ctx, _make_fake_dl_result(success=False, cursor="C1", posts_processed=10), + ) + await db.commit() + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source.id) + )).scalar_one() + assert co.get("_backfill_posts") == 12 # untouched — the ingester owns it + + @pytest.mark.asyncio async def test_backfill_timeout_chunk_reclassified_to_ok( db, db_sync, tmp_path, seed_artist_and_source, @@ -571,7 +733,7 @@ async def test_backfill_timeout_chunk_reclassified_to_ok( success=False, files_downloaded=3, error_type=ErrorType.TIMEOUT, error_message="Download timed out", - stderr="[patreon][debug] Cursor: 03:NEXT:page\n", + cursor="03:NEXT:page", )) event_id = await svc.download_source(source.id) @@ -605,6 +767,32 @@ async def test_tick_mode_when_not_running_leaves_state_untouched( assert (co or {}).get("_backfill_state") is None +@pytest.mark.asyncio +async def test_recovery_mode_selected_and_flag_cleared_on_complete( + db, db_sync, tmp_path, seed_artist_and_source, +): + """plan #697: `_backfill_bypass_seen` alongside a running backfill selects + recovery mode (ingester bypasses the seen-ledger). A clean rc=0 walk + completes the shared lifecycle AND clears the bypass flag so the next tick + honors the ledger again.""" + _artist, source = seed_artist_and_source + source.config_overrides = {"_backfill_state": "running", "_backfill_bypass_seen": True} + source.backfill_runs_remaining = 5 + await db.commit() + + svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + )) + await svc.download_source(source.id) + + assert calls[0]["mode"] == "recovery" + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source.id) + )).scalar_one() + assert co.get("_backfill_state") == "complete" + assert "_backfill_bypass_seen" not in co + + @pytest.mark.asyncio async def test_partial_error_type_maps_to_ok_status( db, db_sync, tmp_path, seed_artist_and_source, @@ -642,6 +830,7 @@ async def test_partial_error_type_maps_to_ok_status( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, fake_result) event_id = await svc.download_source(source.id) ev = (await db.execute( @@ -677,10 +866,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( _make_jpg(f1, split="h") _make_jpg(f2, split="v") - fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + result = _make_fake_dl_result( success=True, written_paths=[str(f1), str(f2)], files_downloaded=2, stdout=f"{f1}\n{f2}\n", - )) + ) + fake_gdl = _fake_gdl_with_result(result) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -713,6 +903,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, result) await svc.download_source(source.id) # Two files attached → two thumbnail enqueues + two ML enqueues, IDs @@ -727,11 +918,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( async def test_releases_db_connections_before_subprocess( db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, ): - """Phase 1's DB connections must be released BEFORE the (up to ~19.5-min - in backfill) gallery-dl subprocess, so they don't idle-die and strand - phase 3 with ConnectionDoesNotExistError (Anduo #40014). Spy on the - async session close and assert it happened before gdl.download runs; - phase 3 must still finalize the event.""" + """Phase 1's DB connections must be released BEFORE the (multi-minute) + phase-2 fetch, so they don't idle-die and strand phase 3 with + ConnectionDoesNotExistError (Anduo #40014). Spy on the async session close + and assert it happened before the phase-2 work (here the native Patreon + ingester) runs; phase 3 must still finalize the event.""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer @@ -747,19 +938,9 @@ async def test_releases_db_connections_before_subprocess( monkeypatch.setattr(db, "close", spy_close) seen = {} - - async def fake_download(url, **k): - seen["closed_before_download"] = closed["async"] - return _make_fake_dl_result(success=True, written_paths=[], stdout="") - - fake_gdl = MagicMock() - fake_gdl.download = fake_download - fake_gdl._compute_run_stats = lambda *a, **k: { - "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, - "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, - } - fake_gdl._extract_errors_warnings = lambda *a, **k: "" - fake_gdl._truncate_log = lambda x, **k: x + fake_gdl = _fake_gdl_with_result( + _make_fake_dl_result(success=True, written_paths=[], stdout="") + ) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -776,9 +957,15 @@ async def test_releases_db_connections_before_subprocess( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + + async def fake_ingest(*, ctx, source_config, skip_value, mode): + seen["closed_before_phase2"] = closed["async"] + return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None + + svc._run_download = fake_ingest event_id = await svc.download_source(source.id) - assert seen["closed_before_download"] is True + assert seen["closed_before_phase2"] is True ev = (await db.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index 5eadee9..76f0997 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -8,7 +8,6 @@ from backend.app.services.gallery_dl import ( ErrorType, GalleryDLService, SourceConfig, - parse_last_cursor, ) @@ -260,7 +259,7 @@ def test_build_config_emits_tick_skip_value(gdl): scans once 20 contiguous archived items are seen.""" from backend.app.services.gallery_dl import TICK_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=TICK_SKIP_VALUE, @@ -273,7 +272,7 @@ def test_build_config_emits_backfill_skip_value(gdl): full post history.""" from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=BACKFILL_SKIP_VALUE, @@ -316,49 +315,6 @@ def test_categorize_tier_limited_wins_over_partial(gdl): return_code=1, stdout=stdout, stderr=stderr, ) assert etype == ErrorType.TIER_LIMITED - - -def test_default_config_forwards_patreon_referer_to_ytdl(gdl): - """Operator-flagged 2026-06-01: Mux video playback restrictions reject - yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The - fix is a static `downloader.ytdl.raw-options.http_headers` block, so - it's enough to assert the keys land in the default config.""" - cfg = gdl._get_default_config() - headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"] - assert headers["Referer"] == "https://www.patreon.com/" - assert headers["Origin"] == "https://www.patreon.com" - - -# --- Cursor-paged backfill (plan #689) ------------------------------------- - - -def test_parse_last_cursor_returns_last_match_across_streams(): - # gallery-dl logs `Cursor: ` per page (debug → stderr); the last - # is the furthest-progressed resume point. - stderr = ( - "[patreon][debug] Cursor: 03:AAAA:bbb\n" - "[urllib3] GET ...\n" - "[patreon][debug] Cursor: 03:CCCC:ddd\n" - ) - assert parse_last_cursor("", stderr) == "03:CCCC:ddd" - - -def test_parse_last_cursor_scans_stdout_too_and_none_when_absent(): - assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ" - assert parse_last_cursor("no marker here", "still nothing") is None - assert parse_last_cursor("", "") is None - - -def test_build_config_patreon_resume_cursor_overrides_default(gdl): - # No resume cursor → PLATFORM_DEFAULTS leaves the log-only `cursor: True`. - fresh = gdl._build_config_for_source( - "patreon", SourceConfig(), "alice", skip_value=True, - ) - assert fresh["extractor"]["patreon"]["cursor"] is True - - # Resume cursor set → gallery-dl restarts the walk from that page. - resumed = gdl._build_config_for_source( - "patreon", SourceConfig(resume_cursor="03:CCCC:ddd"), "alice", - skip_value=True, - ) - assert resumed["extractor"]["patreon"]["cursor"] == "03:CCCC:ddd" +# (The cursor-parsing tests were removed in plan #704: parse_last_cursor is gone +# — the native ingester now carries its checkpoint cursor as a structured +# DownloadResult.cursor field instead of a `Cursor:` log line to scrape.) diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py new file mode 100644 index 0000000..0426147 --- /dev/null +++ b/tests/test_patreon_client.py @@ -0,0 +1,351 @@ +"""PatreonClient tests — parsing only, no network. + +The fixture is loaded from disk and fed directly to the pure parsing methods +(_transform / extract_media / _validate_response / parse_cursor_from_url); no +live requests call is exercised here. +""" + +import json +from pathlib import Path + +import pytest + +from backend.app.services import patreon_client as pc_mod +from backend.app.services.patreon_client import ( + MediaItem, + PatreonAPIError, + PatreonAuthError, + PatreonClient, + PatreonDriftError, + _retry_after_seconds, + parse_cursor_from_url, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json" + + +@pytest.fixture +def response(): + return json.loads(_FIXTURE.read_text()) + + +@pytest.fixture +def client(): + # No cookies file → empty session, but we never make a request in tests. + return PatreonClient(cookies_path=None) + + +def _post_by_id(response, post_id): + for post in response["data"]: + if post["id"] == post_id: + return post + raise AssertionError(f"no post {post_id} in fixture") + + +def test_transform_indexes_included(client, response): + index = client._transform(response) + assert index[("media", "9001")]["file_name"] == "gallery-one.jpg" + assert index[("media", "9003")]["file_name"] == "bonus-pack.zip" + assert index[("campaign", "5555")]["name"] == "Example Creator" + # Unknown keys are absent, not errors. + assert ("media", "0000") not in index + + +def test_extract_gallery_dedups_image_large(client, response): + index = client._transform(response) + post = _post_by_id(response, "1001") + items = client.extract_media(post, index) + + # Two gallery images; the image_large cover shares filehash aaaa... with + # gallery image 9001 → collapses to one, leaving exactly 2 items. + assert len(items) == 2 + assert all(isinstance(m, MediaItem) for m in items) + + kinds = [m.kind for m in items] + assert kinds == ["images", "images"] # image_large deduped away + + filenames = {m.filename for m in items} + assert filenames == {"gallery-one.jpg", "gallery-two.jpg"} + + hashes = {m.filehash for m in items} + assert hashes == {"a" * 32, "b" * 32} + assert all(m.post_id == "1001" for m in items) + + +def test_extract_attachment_postfile_same_file_collapses(client, response): + index = client._transform(response) + post = _post_by_id(response, "1002") + items = client.extract_media(post, index) + + # The attachment (attachments_media 9003) and the post_file are the SAME + # zip — both carry filehash c…. attachments resolve before postfile, so the + # postfile collapses into the attachment by filehash → exactly one survives. + # (postfile-as-a-kind is exercised separately by the video post test.) + assert len(items) == 1 + item = items[0] + assert item.kind == "attachments" + assert item.filename == "bonus-pack.zip" + assert item.filehash == "c" * 32 + + +def test_extract_inline_content_img(client, response): + index = client._transform(response) + post = _post_by_id(response, "1003") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "content" + assert item.filehash == "d" * 32 + # & in the src must be unescaped to & . + assert "&" not in item.url + assert "token=x&v=2" in item.url + + +def test_extract_video_postfile(client, response): + index = client._transform(response) + post = _post_by_id(response, "1004") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "postfile" + assert item.url.startswith("https://stream.mux.com/") + assert item.filehash == "e" * 32 + + +def test_parse_cursor_from_url_extracts_cursor(response): + next_url = response["links"]["next"] + assert parse_cursor_from_url(next_url) == "NEXTCURSOR123" + + +def test_parse_cursor_from_url_none_when_absent(): + assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None + assert parse_cursor_from_url(None) is None + + +def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch): + # Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one + # post, no links.next → stop. Monkeypatch _fetch so no network is touched. + page2 = { + "data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}], + "included": [], + "links": {}, + } + + def fake_fetch(campaign_id, cursor): + return page2 if cursor == "NEXTCURSOR123" else response + + monkeypatch.setattr(client, "_fetch", fake_fetch) + + seen = list(client.iter_posts("5555")) + # 4 posts on page 1 + 1 on page 2. + assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"] + # page_cursor is the cursor that FETCHED the post's page. + cursors = {p["id"]: cur for p, _idx, cur in seen} + assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123" + # included_index is the page's flattened resources, ready for extract_media. + page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001") + assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg" + + +def test_missing_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"included": []}) + + +def test_non_list_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"data": {"id": "1"}}) + + +def test_media_missing_file_name_raises_drift(client): + # A media resource referenced by a gallery relationship but lacking + # file_name is drift from the contract. + post = { + "id": "2000", + "type": "post", + "attributes": {"title": "broken"}, + "relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}}, + } + index = { + ("media", "7777"): { + "download_url": "https://c10.patreonusercontent.com/4/orig/" + + "f" * 32 + + "/x.jpg" + } + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, index) + + +def test_media_referenced_but_absent_raises_drift(client): + post = { + "id": "2001", + "type": "post", + "attributes": {"title": "dangling"}, + "relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}}, + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, {}) + + +# -- HTTP status classification (plan #697 step 4) ------------------------- + + +class _FakeResp: + def __init__(self, status_code, *, json_data=None, raise_json=False, headers=None): + self.status_code = status_code + self._json_data = json_data + self._raise_json = raise_json + self.headers = headers or {} + + def json(self): + if self._raise_json: + raise ValueError("Expecting value: line 1 column 1") + return self._json_data + + +def _client_returning(monkeypatch, resp): + client = PatreonClient(cookies_path=None) + monkeypatch.setattr(client._session, "get", lambda *a, **k: resp) + return client + + +@pytest.mark.parametrize("status", [401, 403]) +def test_fetch_auth_status_raises_auth_error(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + with pytest.raises(PatreonAuthError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == status + + +def test_fetch_non_json_body_is_auth_not_drift(monkeypatch): + # An HTML login/challenge page (non-JSON) means the session expired — auth, + # actionable as "rotate cookies", NOT API drift. + client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True)) + with pytest.raises(PatreonAuthError): + client._fetch("5555", None) + + +@pytest.mark.parametrize("status", [404, 429, 500]) +def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == status + # Not auth — the ingester maps these by status, not to auth_error. + assert not isinstance(ei.value, PatreonAuthError) + + +def test_fetch_ok_returns_payload(monkeypatch): + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []})) + assert client._fetch("5555", None) == {"data": []} + + +# -- verify_auth (credential probe; (True/False/None, message) contract) --- + + +def test_verify_auth_ok_when_page_authenticates(monkeypatch): + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []})) + ok, msg = client.verify_auth("5555") + assert ok is True + assert "valid" in msg.lower() + + +@pytest.mark.parametrize("status", [401, 403]) +def test_verify_auth_false_when_rejected(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + ok, _msg = client.verify_auth("5555") + assert ok is False + + +def test_verify_auth_inconclusive_on_drift(monkeypatch): + # 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a + # credential verdict (our parser is stale, the cookie may be fine). + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []})) + ok, msg = client.verify_auth("5555") + assert ok is None + assert "api shape changed" in msg.lower() + + +def test_verify_auth_inconclusive_on_network(monkeypatch): + import requests + client = PatreonClient(cookies_path=None) + + def _raise(*a, **k): + raise requests.ConnectionError("boom") + monkeypatch.setattr(client._session, "get", _raise) + ok, _msg = client.verify_auth("5555") + assert ok is None + + +# -- 429 backoff + pacing (plan #703) -------------------------------------- + + +def test_retry_after_seconds_honors_numeric_header(): + assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0 + + +def test_retry_after_seconds_exponential_without_header(): + resp = _FakeResp(429) + assert _retry_after_seconds(resp, 1) == 2.0 + assert _retry_after_seconds(resp, 2) == 4.0 + assert _retry_after_seconds(resp, 3) == 8.0 + + +def test_retry_after_seconds_caps(): + assert _retry_after_seconds(_FakeResp(429), 10) == 30.0 + assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0 + + +def _client_with_sequence(monkeypatch, responses): + """A client whose session yields `responses` in order; time.sleep stubbed so + backoff never really sleeps. Returns (client, slept_list).""" + client = PatreonClient(cookies_path=None) + it = iter(responses) + monkeypatch.setattr(client._session, "get", lambda *a, **k: next(it)) + slept: list[float] = [] + monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s)) + return client, slept + + +def test_fetch_retries_then_succeeds_on_429(monkeypatch): + client, slept = _client_with_sequence(monkeypatch, [ + _FakeResp(429, headers={"Retry-After": "5"}), + _FakeResp(200, json_data={"data": []}), + ]) + assert client._fetch("5555", None) == {"data": []} + assert 5.0 in slept # backed off once before the retry + + +def test_fetch_persistent_429_raises_after_retries(monkeypatch): + client, _slept = _client_with_sequence(monkeypatch, [_FakeResp(429)] * 10) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == 429 + assert not isinstance(ei.value, PatreonAuthError) + + +def test_fetch_persistent_429_surfaces_retry_after(monkeypatch): + """plan #708 B1: a terminal 429 carries the server's raw Retry-After seconds + (uncapped here — the cooldown clamps) so the platform cooldown matches it.""" + client, _slept = _client_with_sequence( + monkeypatch, [_FakeResp(429, headers={"Retry-After": "120"})] * 10, + ) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == 429 + assert ei.value.retry_after == 120.0 + + +def test_request_sleep_paces_before_fetch(monkeypatch): + client = PatreonClient(cookies_path=None, request_sleep=1.5) + monkeypatch.setattr( + client._session, "get", + lambda *a, **k: _FakeResp(200, json_data={"data": []}), + ) + slept: list[float] = [] + monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s)) + client._fetch("5555", None) + assert slept == [1.5] diff --git a/tests/test_patreon_contract.py b/tests/test_patreon_contract.py new file mode 100644 index 0000000..1f6a520 --- /dev/null +++ b/tests/test_patreon_contract.py @@ -0,0 +1,72 @@ +"""Patreon API contract test (native ingester build step 4). + +The drift tripwire the plan calls for: if the field-set we SEND or the parser we +resolve responses against drifts from what real Patreon returns, this build goes +RED — instead of the ingester silently importing nothing in production (the exact +failure mode the native ingester exists to escape). + +Two halves: + - the recorded `/api/posts` fixture must still parse end-to-end (no drift), and + - the request params must still carry every field the parser depends on (the + fixture already has the data, so trimming the request would NOT trip the + parse half — this half guards the request shape directly). + +Pure: no network, no DB. +""" + +import json +from pathlib import Path + +from backend.app.services.patreon_client import MediaItem, PatreonClient + +_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json" + +# Pinned media total across the 4 fixture posts: 1001→2 (gallery, image_large +# cover deduped), 1002→1 (attachment+postfile collapse), 1003→1 (inline content +# img), 1004→1 (video postfile). A parser change that drops or doubles media +# trips this. +_EXPECTED_MEDIA_TOTAL = 5 +_KNOWN_KINDS = {"images", "image_large", "attachments", "postfile", "content"} + + +def test_recorded_fixture_parses_end_to_end(): + response = json.loads(_FIXTURE.read_text()) + client = PatreonClient(cookies_path=None) + + client._validate_response(response) # top-level shape must hold + index = client._transform(response) + + total = 0 + for post in response["data"]: + items = client.extract_media(post, index) # must not raise drift + for m in items: + assert isinstance(m, MediaItem) + assert m.url, f"empty url in post {post['id']}" + assert m.filename, f"empty filename in post {post['id']}" + assert m.kind in _KNOWN_KINDS, f"unknown kind {m.kind!r}" + assert m.post_id == post["id"] + total += len(items) + + assert total == _EXPECTED_MEDIA_TOTAL + + +def test_request_field_set_carries_parser_dependencies(): + """The params we SEND must keep requesting the fields the parser resolves + against. Trimming `fields[media]=file_name`, an `include` relationship, or a + `fields[post]` key would make real responses parse to nothing — caught here, + not by the fixture parse above.""" + params = PatreonClient(cookies_path=None)._params("5555", None) + + media_fields = params["fields[media]"] + assert "file_name" in media_fields + assert "image_urls" in media_fields or "download_url" in media_fields + + includes = params["include"] + for rel in ("images", "attachments_media", "media"): + assert rel in includes, f"missing include relationship {rel}" + + post_fields = params["fields[post]"] + for field in ("content", "post_file", "image"): + assert field in post_fields, f"missing post field {field}" + + assert params["filter[campaign_id]"] == "5555" diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py new file mode 100644 index 0000000..0522807 --- /dev/null +++ b/tests/test_patreon_downloader.py @@ -0,0 +1,519 @@ +"""Unit tests for PatreonDownloader (build step 2b). + +No network, no real subprocess. The HTTP layer is stubbed via the `session` +seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp +via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import requests + +from backend.app.services import patreon_downloader as pd_mod +from backend.app.services.patreon_client import MediaItem +from backend.app.services.patreon_downloader import ( + MediaOutcome, + PatreonDownloader, + _sanitize, +) +from backend.app.utils.sidecar import find_sidecar + +# Minimal valid PNG so the file_validator passes on the happy path. +_PNG_HEAD = b"\x89PNG\r\n\x1a\n" +_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82" +_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL + + +class _FakeResponse: + def __init__(self, payload: bytes, status_code: int = 200, headers=None): + self._payload = payload + self.status_code = status_code + self.headers = headers or {} + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"HTTP {self.status_code}") + return None + + def iter_content(self, chunk_size=65536): + for i in range(0, len(self._payload), chunk_size): + yield self._payload[i : i + chunk_size] + + +class _FakeSession: + """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES).""" + + def __init__(self, payloads: dict[str, bytes] | None = None): + self.payloads = payloads or {} + self.calls: list[str] = [] + + def get(self, url, stream=False, timeout=None, headers=None): + self.calls.append(url) + return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) + + +class _SeqSession: + """Returns the queued items in order (for retry tests). An Exception item is + RAISED by get() instead of returned (simulates a transport error).""" + + def __init__(self, responses): + self._responses = list(responses) + self.calls: list[str] = [] + + def get(self, url, stream=False, timeout=None, headers=None): + self.calls.append(url) + item = self._responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +class _FlakyResp: + """A streaming response that yields its payload then optionally raises a + mid-stream transport error (for the Range-resume tests, plan #708 B5).""" + + headers: dict = {} + + def __init__(self, payload, status_code=200, die=False): + self._payload = payload + self.status_code = status_code + self._die = die + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=65536): + yield self._payload + if self._die: + raise requests.exceptions.ChunkedEncodingError("cut mid-stream") + + +def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): + return { + "id": post_id, + "attributes": { + "title": title, + "content": "

body

", + "published_at": published, + "url": f"https://www.patreon.com/posts/{post_id}", + }, + } + + +def _img(name, post_id="1001", url=None): + return MediaItem( + url=url or f"https://cdn.patreon.com/{name}", + filename=name, + kind="images", + filehash=None, + post_id=post_id, + ) + + +def _downloader(tmp_path: Path, session=None, validate=True): + return PatreonDownloader( + images_root=tmp_path, + cookies_path=None, + validate=validate, + session=session or _FakeSession(), + ) + + +def test_path_layout_two_images(tmp_path): + dl = _downloader(tmp_path) + items = [_img("media1.png"), _img("media2.png")] + outcomes = dl.download_post(_post(), items, "artist-x") + + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + p1 = post_dir / "01_media1.png" + p2 = post_dir / "02_media2.png" + assert p1.is_file() + assert p2.is_file() + assert [o.status for o in outcomes] == ["downloaded", "downloaded"] + assert outcomes[0].path == p1 + assert p1.read_bytes() == _PNG_BYTES + + +def test_dir_sanitized(tmp_path): + dl = _downloader(tmp_path) + post = _post(title='bad/name:with*chars?') + dl.download_post(post, [_img("a.png")], "artist-x") + patreon_dir = tmp_path / "artist-x" / "patreon" + children = list(patreon_dir.iterdir()) + assert len(children) == 1 + name = children[0].name + assert "/" not in name + assert not any(c in name for c in '<>:"\\|?*') + + +def test_no_date_prefix_when_unparseable(tmp_path): + dl = _downloader(tmp_path) + post = _post(published="not-a-date") + dl.download_post(post, [_img("a.png")], "artist-x") + patreon_dir = tmp_path / "artist-x" / "patreon" + children = [c.name for c in patreon_dir.iterdir()] + assert children == ["1001_My Post Title"] + + +def test_sidecar_written_and_findable(tmp_path): + dl = _downloader(tmp_path) + outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") + media_path = outcomes[0].path + + sidecar = find_sidecar(media_path) + assert sidecar is not None + assert sidecar.name == "01_media1.json" + + import json + + data = json.loads(sidecar.read_text()) + assert data["category"] == "patreon" + assert data["id"] == "1001" + assert data["title"] == "My Post Title" + assert data["url"] == "https://www.patreon.com/posts/1001" + + +def test_skip_seen(tmp_path): + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post( + _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True + ) + assert outcomes[0].status == "skipped_seen" + assert outcomes[0].path is None + assert session.calls == [] + # No file written anywhere. + assert not (tmp_path / "artist-x").exists() + + +def test_skip_disk(tmp_path): + dl = _downloader(tmp_path) + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + post_dir.mkdir(parents=True) + existing = post_dir / "01_media1.png" + existing.write_bytes(b"already here") + + outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") + assert outcomes[0].status == "skipped_disk" + assert outcomes[0].path == existing + assert existing.read_bytes() == b"already here" # untouched + + +def test_video_routes_to_ytdlp(tmp_path, monkeypatch): + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + captured = {} + + def fake_ytdlp(url, dest, headers): + captured["url"] = url + captured["headers"] = headers + out = Path(dest).with_suffix(".mp4") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"video-bytes") + return out + + monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp) + + video = MediaItem( + url="https://stream.mux.com/abc123.m3u8", + filename="clip.mp4", + kind="postfile", + filehash=None, + post_id="1001", + ) + outcomes = dl.download_post(_post(), [video], "artist-x") + + assert captured["url"] == "https://stream.mux.com/abc123.m3u8" + assert captured["headers"]["Referer"] == "https://www.patreon.com/" + assert captured["headers"]["Origin"] == "https://www.patreon.com" + assert session.calls == [] # NOT the plain-GET path + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == b"video-bytes" + # Sidecar written next to the actual (remuxed) output. + assert find_sidecar(outcomes[0].path) is not None + + +def _video_item(): + return MediaItem( + url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4", + kind="postfile", filehash=None, post_id="1001", + ) + + +def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch): + """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within + the pass; a later success yields a downloaded outcome. (The GET path already + retried transients; the video path didn't until #8 closed the gap.)""" + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) + out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4")) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"video-bytes") + return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) + + dl = _downloader(tmp_path, session=_FakeSession()) + outcomes = dl.download_post(_post(), [_video_item()], "artist-x") + assert calls["n"] == 2 # retried once after the transient blip + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == b"video-bytes" + + +def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch): + """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT — + yt-dlp already did its own network retries — so we don't re-spawn; the item + errors straight to the per-item/dead-letter path.""" + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone") + + monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) + + dl = _downloader(tmp_path, session=_FakeSession()) + outcomes = dl.download_post(_post(), [_video_item()], "artist-x") + assert calls["n"] == 1 # no retry on a permanent failure + assert outcomes[0].status == "error" + + +def test_one_failure_isolated(tmp_path): + class _BoomSession(_FakeSession): + def get(self, url, stream=False, timeout=None, headers=None): + self.calls.append(url) + if "bad" in url: + raise RuntimeError("network boom") + return _FakeResponse(_PNG_BYTES) + + dl = _downloader(tmp_path, session=_BoomSession()) + items = [ + _img("good.png", url="https://cdn.patreon.com/good.png"), + _img("bad.png", url="https://cdn.patreon.com/bad.png"), + ] + outcomes = dl.download_post(_post(), items, "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[1].status == "error" + assert "boom" in outcomes[1].error + + +def test_validation_quarantines_corrupt(tmp_path): + # A .png whose bytes are not a valid PNG → validator fails → "quarantined" + # outcome (distinct from a download error) carrying the dest path + + # quarantine move; with validate=False it would pass. plan #704 (#4). + bad = b"not a real png" + session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad}) + dl = _downloader(tmp_path, session=session, validate=True) + item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png") + outcomes = dl.download_post(_post(), [item], "artist-x") + assert outcomes[0].status == "quarantined" + assert outcomes[0].path is not None + assert "_quarantine" in str(outcomes[0].path) + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + assert not (post_dir / "01_corrupt.png").exists() + quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon" + assert quarantine.is_dir() + assert any(quarantine.iterdir()) + + +def test_quarantine_writes_provenance_sidecar(tmp_path): + """A1 parity fix: the native path now writes the same `.quarantine.json` + provenance sidecar gallery-dl writes (it skipped it before the shared + file_validator.quarantine_file helper).""" + import json + + bad = b"not a real png" + url = "https://cdn.patreon.com/corrupt.png" + session = _FakeSession({url: bad}) + dl = _downloader(tmp_path, session=session, validate=True) + outcomes = dl.download_post(_post(), [_img("corrupt.png", url=url)], "artist-x") + + dest = outcomes[0].path + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + assert sidecar.is_file() + meta = json.loads(sidecar.read_text()) + assert meta["source_url"] == url + assert meta["platform"] == "patreon" + assert meta["artist_slug"] == "artist-x" + assert meta["reason"] + + +def test_validation_off_keeps_bytes(tmp_path): + bad = b"not a real png" + session = _FakeSession({"https://cdn.patreon.com/x.png": bad}) + dl = _downloader(tmp_path, session=session, validate=False) + item = _img("x.png", url="https://cdn.patreon.com/x.png") + outcomes = dl.download_post(_post(), [item], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == bad + + +def test_sanitize_helper(): + assert _sanitize('a/b') == "a_b" + assert _sanitize('x<>:"|?*y') == "x_______y" + assert _sanitize("trail. ") == "trail" + assert _sanitize("") == "_" + + +def test_media_outcome_shape(): + o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None) + assert o.status == "downloaded" + assert o.path == Path("/tmp/x") + + +# -- pacing + 429 backoff (plan #703) -------------------------------------- + + +def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + rate_limit=2.0, session=_FakeSession(), + ) + dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x") + # One pacing sleep per actual download (two media downloaded). + assert slept == [2.0, 2.0] + + +def test_skips_do_not_pace(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + rate_limit=2.0, session=_FakeSession(), + ) + # is_seen → skipped_seen, never reaches the download/pacing path. + dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True) + assert slept == [] + + +def test_media_429_retried_then_succeeds(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + session = _SeqSession([ + _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert 4.0 in slept # backed off before the retry + assert len(session.calls) == 2 + + +def _dl(tmp_path, session, monkeypatch): + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None) + return PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + + +def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch): + """plan #705 #8: a connection blip on the GET is retried in-place, not a + terminal error that waits for the next walk.""" + session = _SeqSession([ + requests.ConnectionError("connection reset"), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert len(session.calls) == 2 + + +def test_5xx_retried_within_pass(tmp_path, monkeypatch): + session = _SeqSession([ + _FakeResponse(b"", status_code=503), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert len(session.calls) == 2 + + +def test_404_fails_fast_no_retry(tmp_path, monkeypatch): + """A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately + (and the ingester's dead-letter ledger takes over across walks).""" + session = _SeqSession([ + _FakeResponse(b"", status_code=404), + _FakeResponse(_PNG_BYTES, status_code=200), # would succeed if it retried + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "error" + assert len(session.calls) == 1 # no retry on a permanent failure + + +def test_exhausted_transient_becomes_error(tmp_path, monkeypatch): + session = _SeqSession([requests.Timeout("t")] * 10) # always times out + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "error" + + +def test_partial_download_resumes_via_range(tmp_path, monkeypatch): + """plan #708 B5: a mid-download transport cut resumes from the bytes already + on disk via a Range request, not a refetch from zero.""" + full = bytes(i % 256 for i in range(1000)) + split = 400 + + class _ResumeSession: + def __init__(self): + self.headers_seen = [] + + def get(self, url, stream=False, timeout=None, headers=None): + self.headers_seen.append(headers) + if len(self.headers_seen) == 1: + return _FlakyResp(full[:split], die=True) # partial then cut + start = int(headers["Range"].split("=")[1].rstrip("-")) + return _FlakyResp(full[start:], status_code=206) + + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) + session = _ResumeSession() + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == full + assert session.headers_seen[0] is None # first GET: no Range + assert session.headers_seen[1] == {"Range": "bytes=400-"} # resume from offset + + +def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch): + """plan #708 B5: if the server ignores Range (200, not 206) on the resume, we + truncate and start clean — never append into a corrupt double-write.""" + full = b"ABCD" * 250 # 1000 bytes + split = 400 + + class _NoRangeSession: + def __init__(self): + self.n = 0 + + def get(self, url, stream=False, timeout=None, headers=None): + self.n += 1 + if self.n == 1: + return _FlakyResp(full[:split], die=True) + return _FlakyResp(full, status_code=200) # ignores Range, whole file + + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_NoRangeSession(), + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py new file mode 100644 index 0000000..95da4bd --- /dev/null +++ b/tests/test_patreon_ingester.py @@ -0,0 +1,711 @@ +"""PatreonIngester tests (native ingester build step 3). + +The HTTP client and the media downloader are stubbed (injected seams), so these +exercise the ingester's own logic — mode behavior, the two-tier skip, cursor +emission, the budget cutoff, and the Postgres seen-ledger — without network or a +real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so +the tier-1 skip and the idempotent mark-seen run against actual rows. +""" + +import pytest +from sqlalchemy import func, select +from sqlalchemy.orm import sessionmaker + +from backend.app.models import ( + Artist, + PatreonFailedMedia, + PatreonSeenMedia, + Source, +) +from backend.app.services.gallery_dl import ErrorType +from backend.app.services.patreon_client import ( + MediaItem, + PatreonAPIError, + PatreonAuthError, + PatreonDriftError, +) +from backend.app.services.patreon_downloader import MediaOutcome +from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key + +pytestmark = pytest.mark.integration + + +# --- fakes ---------------------------------------------------------------- + + +def _media(post_id, n, *, filehash=None, kind="images"): + # Deterministic 32-hex-ish key, unique per (post_id, n). + fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32] + return MediaItem( + url=f"https://cdn.patreon.com/{fh}/{n}.jpg", + filename=f"{n}.jpg", + kind=kind, + filehash=fh, + post_id=post_id, + ) + + +class _FakeClient: + """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post + is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift.""" + + def __init__(self, pages, raise_exc=None): + self._pages = pages + self._raise_exc = raise_exc + self.consumed_posts = 0 + + def iter_posts(self, campaign_id, cursor=None): + if self._raise_exc is not None: + raise self._raise_exc + for page_cursor, posts in self._pages: + for post_id, media in posts: + self.consumed_posts += 1 + yield {"id": post_id, "_media": media}, {}, page_cursor + + def extract_media(self, post, included_index): + return post["_media"] + + def post_meta(self, post): + return {"title": post.get("id"), "date": None} + + +class _FakeDownloader: + """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in + `on_disk` report skipped_disk (tier-2); media in `quarantine` report + quarantined; everything else downloads.""" + + def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None): + self.tmp_path = tmp_path + self.on_disk = set(on_disk or ()) + self.quarantine = set(quarantine or ()) + self.error = set(error or ()) + self.download_calls = 0 + + def download_post(self, post, media_items, artist_slug, *, is_seen): + outcomes = [] + for m in media_items: + if is_seen(m): + outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None)) + elif _ledger_key(m) in self.on_disk: + p = self.tmp_path / f"{m.post_id}_{m.filename}" + outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None)) + elif _ledger_key(m) in self.error: + self.download_calls += 1 + outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom")) + elif _ledger_key(m) in self.quarantine: + self.download_calls += 1 + q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}" + outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image")) + else: + self.download_calls += 1 + p = self.tmp_path / f"{m.post_id}_{m.filename}" + p.write_bytes(b"x") + outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None)) + return outcomes + + +@pytest.fixture +async def source_id(db): + artist = Artist(name="Ingest", slug="ingest") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/ingest", enabled=True, config_overrides={}, + ) + db.add(source) + await db.commit() + return source.id + + +def _ingester(sync_engine, tmp_path, client, downloader): + factory = sessionmaker(sync_engine, expire_on_commit=False) + return PatreonIngester( + images_root=tmp_path, cookies_path=None, session_factory=factory, + client=client, downloader=downloader, + ) + + +def _count_ledger(sync_engine, source_id): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + return s.execute( + select(func.count(PatreonSeenMedia.id)).where( + PatreonSeenMedia.source_id == source_id + ) + ).scalar_one() + + +# --- tick ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path): + m1, m2 = _media("p1", 1), _media("p1", 2) + client = _FakeClient([(None, [("p1", [m1, m2])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + + assert result.success is True + assert result.return_code == 0 + assert result.files_downloaded == 2 + assert len(result.written_paths) == 2 + # plan #704: structured run_stats carry the real counts. + assert result.run_stats["downloaded_count"] == 2 + assert result.posts_processed == 1 + assert _count_ledger(sync_engine, source_id) == 2 + + +@pytest.mark.asyncio +async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path): + """plan #704 (#4): a media that fails validation is reported as quarantined — + a real files_quarantined count + paths + run_stats — not a blank 0, and is + NOT written or marked seen.""" + m1, m2 = _media("p1", 1), _media("p1", 2) + client = _FakeClient([(None, [("p1", [m1, m2])])]) + downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)}) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 1 # only m1 + assert result.files_quarantined == 1 # m2 + assert len(result.quarantined_paths) == 1 + assert result.run_stats["quarantined_count"] == 1 + assert result.run_stats["downloaded_count"] == 1 + assert len(result.written_paths) == 1 # quarantined NOT written + # Quarantined media is NOT marked seen (a fixed file may be re-fetched). + assert _count_ledger(sync_engine, source_id) == 1 + + +@pytest.mark.asyncio +async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + # Pre-seed the ledger with m1's key. + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 0 + assert downloader.download_calls == 0 + + +@pytest.mark.asyncio +async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path): + # Three posts, one already-seen media each; threshold 2 → stop before the 3rd. + seen_media = [_media(f"p{i}", 1) for i in range(1, 4)] + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + for m in seen_media: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id)) + s.commit() + + pages = [(None, [(m.post_id, [m]) for m in seen_media])] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", seen_threshold=2, + ) + assert result.success is True + # Stopped after the 2nd contiguous seen item — never consumed the 3rd post. + assert client.consumed_posts == 2 + + +# --- backfill ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path): + pages = [ + (None, [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete". + assert result.return_code == 0 + assert result.error_type is None + assert result.files_downloaded == 2 + # The page-2 cursor is carried structurally for checkpointing (plan #704). + assert result.cursor == "CUR2" + assert result.posts_processed == 2 + assert result.run_stats["downloaded_count"] == 2 + + +@pytest.mark.asyncio +async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db): + """plan #705 #6: the cursor is persisted to the DB at each page boundary + DURING the walk (not just at the end via phase 3), so a worker SIGKILL + mid-chunk resumes near the crash. After the walk the source carries the last + page's cursor — written by the ingester, not phase 3 (which the unit run + never reaches).""" + pages = [ + (None, [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ("CUR3", [("p3", [_media("p3", 1)])]), + ] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert co.get("_backfill_cursor") == "CUR3" + + +@pytest.mark.asyncio +async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db): + """plan #704 #5: the ingester writes a monotonic _backfill_posts absolute + DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the + re-walked resume page so the count doesn't inflate across chunks.""" + # Resume from CUR2 (a prior chunk left off here): its page is re-walked and + # must NOT be re-counted. posts_base=4 stands in for the prior chunks. + pages = [ + ("CUR2", [("p1", [_media("p1", 1)])]), # the resumed page — not counted + ("CUR3", [("p2", [_media("p2", 1)])]), # net-new + ("CUR4", [("p3", [_media("p3", 1)])]), # net-new + ] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + resume_cursor="CUR2", posts_base=4, + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + # 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded. + assert co.get("_backfill_posts") == 6 + + +@pytest.mark.asyncio +async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db): + """A tick has no resumable backfill state, so it must not write _backfill_posts + (the badge is a backfill/recovery concept).""" + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]), + _FakeDownloader(tmp_path), + ) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert "_backfill_posts" not in (co or {}) + + +@pytest.mark.asyncio +async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db): + """A tick has no resumable backfill state, so it must not write a cursor.""" + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]), + _FakeDownloader(tmp_path), + ) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert "_backfill_cursor" not in (co or {}) + + +@pytest.mark.asyncio +async def test_backfill_budget_cut_returns_partial_with_progress( + source_id, sync_engine, tmp_path, monkeypatch, +): + # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget). + # run() lives in ingest_core now, so patch the clock there (the same global + # time module object, but we reference it through the module that uses it). + import backend.app.services.ingest_core as core + ticks = iter([0.0, 10.0, 200.0, 250.0]) + last = [0.0] + + def fake_monotonic(): + try: + last[0] = next(ticks) + except StopIteration: + pass + return last[0] + + monkeypatch.setattr(core.time, "monotonic", fake_monotonic) + + pages = [ + ("CUR1", [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + resume_cursor=None, time_budget_seconds=100.0, + ) + assert result.error_type == ErrorType.PARTIAL + assert result.return_code == -1 + assert result.files_downloaded == 1 # post1 downloaded before the cut + assert downloader.download_calls == 1 # post2's body never ran + # The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted, + # then the budget check broke before processing it); the next chunk resumes + # there. Structured (plan #704), matching the old parse_last_cursor(last). + assert result.cursor == "CUR2" + + +@pytest.mark.asyncio +async def test_preview_counts_new_media_without_downloading( + source_id, sync_engine, tmp_path, +): + """plan #708 B4: preview walks + counts media not already seen/dead, downloads + nothing, and samples only posts that have new media.""" + m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2) + # Seed m1 as already-seen → only p2's two media are "new". + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.preview(source_id, "c1") + assert result["total_new"] == 2 # p2's m2 + m3 (m1 already seen) + assert result["posts_scanned"] == 2 + assert result["has_more"] is False + assert downloader.download_calls == 0 # dry-run — nothing downloaded + # The sample lists only posts WITH new media (p2), not the all-seen p1. + assert [row["new"] for row in result["sample"]] == [2] + + +@pytest.mark.asyncio +async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path): + """plan #708 B4: preview stops after page_limit pages and flags has_more.""" + pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + result = ing.preview(source_id, "c1", page_limit=2) + assert result["pages_scanned"] == 2 + assert result["posts_scanned"] == 2 # one post per page, 2 pages + assert result["has_more"] is True + + +@pytest.mark.asyncio +async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db): + """plan #708 B4: _still_running reflects the source's `_backfill_state` — the + flag an operator Stop pops to cancel a live chunk.""" + ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path)) + assert ing._still_running(source_id) is False # absent → not running + src = (await db.execute(select(Source).where(Source.id == source_id))).scalar_one() + src.config_overrides = {"_backfill_state": "running"} + await db.commit() + assert ing._still_running(source_id) is True + + +@pytest.mark.asyncio +async def test_backfill_stops_when_operator_cancels_mid_walk( + source_id, sync_engine, tmp_path, +): + """plan #708 B4: with the running state latched, a later page boundary that + finds it gone (Stop) bails the chunk with PARTIAL instead of finishing.""" + pages = [ + ("CUR2", [("p1", [_media("p1", 1)])]), + ("CUR3", [("p2", [_media("p2", 1)])]), + ("CUR4", [("p3", [_media("p3", 1)])]), + ] + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), downloader) + + # Latch on the page-2 boundary (running), then "Stop" before page 3. + calls = {"n": 0} + + def fake_still_running(_source_id): + calls["n"] += 1 + return calls["n"] == 1 + + ing._still_running = fake_still_running + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert result.error_type == ErrorType.PARTIAL + assert "Stopped by operator" in result.error_message + assert downloader.download_calls == 1 # only p1 ran; cancelled before p2 + + +# --- recovery ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh + # pHash evaluation under the current threshold. + assert result.files_downloaded == 1 + assert downloader.download_calls == 1 + + +@pytest.mark.asyncio +async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + client = _FakeClient([(None, [("p1", [m1])])]) + # File still on disk (a kept image) → tier-2 spares it even under recovery. + downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + assert result.files_downloaded == 0 + assert downloader.download_calls == 0 + # Disk-skip still reconciles the ledger so a later tick skips at tier-1. + assert _count_ledger(sync_engine, source_id) == 1 + + +# --- dead-letter ledger (plan #705 #7) ------------------------------------ + + +def _count_failed(sync_engine, source_id): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + return s.execute( + select(func.count(PatreonFailedMedia.id)).where( + PatreonFailedMedia.source_id == source_id + ) + ).scalar_one() + + +def _seed_failed(sync_engine, source_id, key, attempts): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts)) + s.commit() + + +@pytest.mark.asyncio +async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path): + """A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk + skips it (no download attempt) and counts it dead-lettered.""" + from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD + + m1 = _media("p1", 1) + + def _run(): + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([(None, [("p1", [m1])])]), + _FakeDownloader(tmp_path, error={_ledger_key(m1)}), + ) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + return result, ing.downloader + + # Fail it up to the threshold — each run attempts the download and errors. + for _ in range(DEAD_LETTER_THRESHOLD): + result, dl = _run() + assert dl.download_calls == 1 + assert _count_failed(sync_engine, source_id) == 1 + + # Now attempts == threshold → dead: the next walk skips it without trying. + result, dl = _run() + assert dl.download_calls == 0 + assert result.run_stats["dead_lettered_count"] == 1 + + +@pytest.mark.asyncio +async def test_recovery_reattempts_dead_lettered_and_clears_on_success( + source_id, sync_engine, tmp_path, +): + """Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean + download clears the row (it recovered).""" + from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD + + m1 = _media("p1", 1) + _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD) + + downloader = _FakeDownloader(tmp_path) # succeeds + ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + assert downloader.download_calls == 1 # re-attempted despite being dead + assert result.files_downloaded == 1 + assert _count_failed(sync_engine, source_id) == 0 # cleared on success + + +@pytest.mark.asyncio +async def test_clean_download_clears_below_threshold_failure( + source_id, sync_engine, tmp_path, +): + """A media with a sub-threshold failure that now downloads cleanly (tick) has + its dead-letter row cleared.""" + m1 = _media("p1", 1) + _seed_failed(sync_engine, source_id, _ledger_key(m1), 1) # not yet dead + + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 1 + assert _count_failed(sync_engine, source_id) == 0 + + +# --- ledger + drift ------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + # Same media downloaded twice across two runs — the second marks seen again. + ing._mark_seen(source_id, [(_ledger_key(m1), "p1")]) + ing._mark_seen(source_id, [(_ledger_key(m1), "p1")]) + assert _count_ledger(sync_engine, source_id) == 1 + + +def _run_with_client_error(source_id, sync_engine, tmp_path, exc): + client = _FakeClient([], raise_exc=exc) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + return ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + + +@pytest.mark.asyncio +async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonDriftError("missing top-level 'data' key"), + ) + assert result.success is False + assert result.error_type == ErrorType.API_DRIFT + assert "Patreon API changed" in (result.error_message or "") + + +@pytest.mark.asyncio +async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAuthError("HTTP 403 — auth rejected", status_code=403), + ) + assert result.success is False + assert result.error_type == ErrorType.AUTH_ERROR + + +@pytest.mark.asyncio +async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("HTTP 429", status_code=429), + ) + assert result.error_type == ErrorType.RATE_LIMITED + + +@pytest.mark.asyncio +async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("HTTP 404", status_code=404), + ) + assert result.error_type == ErrorType.NOT_FOUND + + +@pytest.mark.asyncio +async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("connection reset"), # no status_code → transport + ) + assert result.error_type == ErrorType.NETWORK_ERROR + + +def test_ledger_key_video_has_no_filehash(): + video = MediaItem( + url="https://stream.mux.com/abc.m3u8", filename="clip.mp4", + kind="postfile", filehash=None, post_id="p9", + ) + assert _ledger_key(video) == "p9:clip.mp4" + + +# --- verify_patreon_credential (the verify counterpart, plan #697) --------- + + +@pytest.mark.asyncio +async def test_verify_patreon_credential_unresolvable_is_inconclusive(): + from backend.app.services.patreon_ingester import verify_patreon_credential + + # No campaign id resolvable from a non-Patreon URL → can't test → None. + ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {}) + assert ok is None + assert "campaign id" in msg.lower() + + +@pytest.mark.asyncio +async def test_verify_patreon_credential_delegates_to_client(monkeypatch): + import backend.app.services.patreon_ingester as mod + + async def _fake_resolve(url, cookies_path, overrides): + return "123", None + monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve) + monkeypatch.setattr( + mod.PatreonClient, "verify_auth", + lambda self, campaign_id: (True, f"ok:{campaign_id}"), + ) + ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {}) + assert ok is True + assert msg == "ok:123" diff --git a/tests/test_patreon_resolver.py b/tests/test_patreon_resolver.py index ecfe165..bbf40e4 100644 --- a/tests/test_patreon_resolver.py +++ b/tests/test_patreon_resolver.py @@ -2,7 +2,10 @@ from unittest.mock import MagicMock, patch import pytest -from backend.app.services.patreon_resolver import resolve_campaign_id +from backend.app.services.patreon_resolver import ( + resolve_campaign_id, + resolve_campaign_id_for_source, +) @pytest.mark.asyncio @@ -86,3 +89,48 @@ async def test_loads_cookies_file_when_present(tmp_path): assert result == "9" _, kwargs = mock_get.call_args assert kwargs.get("cookies") is not None + + +# -- resolve_campaign_id_for_source (shared by download + verify) ---------- + + +@pytest.mark.asyncio +async def test_for_source_uses_cached_override_without_lookup(): + with patch("backend.app.services.patreon_resolver.requests.get") as mock_get: + cid, resolved = await resolve_campaign_id_for_source( + "https://patreon.com/alice", None, {"patreon_campaign_id": "999"}, + ) + assert cid == "999" + assert resolved is None # cached → no newly-resolved id to cache + mock_get.assert_not_called() + + +@pytest.mark.asyncio +async def test_for_source_extracts_id_url_without_lookup(): + with patch("backend.app.services.patreon_resolver.requests.get") as mock_get: + cid, resolved = await resolve_campaign_id_for_source( + "https://www.patreon.com/id:4242", None, {}, + ) + assert cid == "4242" + assert resolved is None + mock_get.assert_not_called() + + +@pytest.mark.asyncio +async def test_for_source_resolves_vanity_and_reports_it(): + fake = MagicMock() + fake.status_code = 200 + fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]} + with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake): + cid, resolved = await resolve_campaign_id_for_source( + "https://patreon.com/alice", None, {}, + ) + assert cid == "777" + assert resolved == "777" # newly resolved → caller caches it + + +@pytest.mark.asyncio +async def test_for_source_unresolvable_returns_none(): + cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {}) + assert cid is None + assert resolved is None diff --git a/tests/test_patreon_seen_media.py b/tests/test_patreon_seen_media.py new file mode 100644 index 0000000..e80e9bf --- /dev/null +++ b/tests/test_patreon_seen_media.py @@ -0,0 +1,73 @@ +import pytest +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError + +from backend.app.models import Artist, PatreonSeenMedia, Source + +pytestmark = pytest.mark.integration + + +async def _source(db): + artist = Artist(name="Seen Ledger Artist", slug="seen-ledger-artist") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, + platform="patreon", + url="https://www.patreon.com/seenledger", + ) + db.add(source) + await db.flush() + return source.id + + +@pytest.mark.asyncio +async def test_patreon_seen_media_dedup(db): + source_id = await _source(db) + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="123456", + ) + ) + await db.flush() + + seen_at = await db.scalar( + select(PatreonSeenMedia.seen_at).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert seen_at is not None + + # A second sighting of the same (source_id, filehash) is rejected by the + # unique constraint. Wrap in a savepoint so the IntegrityError doesn't + # poison the outer transaction. + with pytest.raises(IntegrityError): + async with db.begin_nested(): + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="654321", + ) + ) + await db.flush() + + # Same filehash but a different post_id sentinel still inserts: the + # dedup axis is (source_id, filehash), not post_id. + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="video:123456:7890", + post_id="123456", + ) + ) + await db.flush() + + count = await db.scalar( + select(func.count(PatreonSeenMedia.id)).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert count == 2 diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 7e53850..7bceea8 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -225,6 +225,48 @@ async def test_stop_backfill_returns_to_idle(db): assert "_backfill_state" not in (co or {}) +@pytest.mark.asyncio +async def test_start_recovery_arms_bypass_flag(db): + """Plan #697: start_recovery arms the backfill state machine PLUS the + _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge + can label it 'Recovering'. Clears any prior checkpoint state.""" + from backend.app.services.source_service import BACKFILL_MAX_CHUNKS + + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + # Simulate a prior walk's posts counter — start must clear it (plan #704). + await db.execute( + Source.__table__.update().where(Source.id == rec.id).values( + config_overrides={"_backfill_posts": 42}, + ) + ) + await db.commit() + + updated = await svc.start_recovery(rec.id) + assert updated.backfill_state == "running" + assert updated.backfill_bypass_seen is True + assert updated.backfill_posts == 0 # cleared for a fresh walk + assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS + + co = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert co.get("_backfill_bypass_seen") is True + assert "_backfill_posts" not in co + + # Stop clears the bypass flag too (shared lifecycle). + stopped = await svc.stop_backfill(rec.id) + assert stopped.backfill_bypass_seen is False + co2 = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert "_backfill_bypass_seen" not in (co2 or {}) + + @pytest.mark.asyncio async def test_start_backfill_raises_when_source_missing(db): svc = SourceService(db)