FabledCurator/backend/app/services/download_result.py

"""What a download walk reports, whichever backend ran it.

The native ingesters (Patreon, SubscribeStar, Discord) and the gallery-dl
subprocess (HentaiFoundry) both return a `DownloadResult`, classify failures
with the same `ErrorType`s and build `run_stats` through `make_run_stats`, so
phase 3, the source's status chip and the Logs UI read one shape from either.
"""

from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass, field
from enum import StrEnum


class ErrorType(StrEnum):
    AUTH_ERROR = "auth_error"
    RATE_LIMITED = "rate_limited"
    NOT_FOUND = "not_found"
    ACCESS_DENIED = "access_denied"
    NETWORK_ERROR = "network_error"
    NO_NEW_CONTENT = "no_new_content"
    TIER_LIMITED = "tier_limited"
    TIMEOUT = "timeout"
    HTTP_ERROR = "http_error"
    UNSUPPORTED_URL = "unsupported_url"
    VALIDATION_FAILED = "validation_failed"
    # Native Patreon ingester only (plan #697): a response parsed as JSON but
    # didn't match the JSON:API shape the ingester depends on (missing `data`,
    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
    # is updating the ingester's field-set/parser, not rotating credentials. The
    # contract test guards against silently shipping this.
    API_DRIFT = "api_drift"
    # Run made real progress (downloaded ≥1 file) but did not finish in the
    # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
    # mapping classifies this as "ok" because the next tick continues.
    PARTIAL = "partial"
    UNKNOWN_ERROR = "unknown_error"


@dataclass
class DownloadResult:
    success: bool
    url: str
    artist_slug: str
    platform: str
    files_downloaded: int = 0
    files_quarantined: int = 0
    quarantined_paths: list[str] = field(default_factory=list)
    written_paths: list[str] = field(default_factory=list)
    # Native ingester only: post-only sidecar paths for posts that have NO
    # downloadable media (pure-text posts), so the importer can still upsert the
    # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
    # Importer.upsert_post_record (keyed on external_post_id → updates, never
    # doubles, the same Post a media import would create).
    post_record_paths: list[str] = field(default_factory=list)
    # Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
    # pairs for already-present media whose ImageRecord.source_filehash should be
    # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
    # on the gallery-dl path and outside recapture.
    relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
    # Native ingester only: one entry per `written_paths` entry, the backfill
    # cursor of the page that file came from. Phase 3 resumes the next chunk
    # there when it leaves files unimported at its deadline. Empty on gallery-dl.
    written_cursors: list[str | None] = field(default_factory=list)
    stdout: str = ""
    stderr: str = ""
    return_code: int = 0
    error_type: ErrorType | None = None
    error_message: str | None = None
    duration_seconds: float = 0.0
    started_at: str | None = None
    completed_at: str | None = None
    # Plan #704 — structured fields the NATIVE ingester populates directly (None
    # on the gallery-dl path, which keeps the regex-over-stdout route). When set,
    # phase 3 reads these instead of scraping the text it would otherwise have to
    # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
    # backfill checkpoint the ingester knows exactly (no parse_last_cursor).
    run_stats: dict | None = None
    cursor: str | None = None
    posts_processed: int = 0
    # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
    # the platform cooldown matches the hint instead of a flat default. None when
    # unknown (no header, or not a rate-limit failure).
    retry_after_seconds: float | None = None
    # Native ingester only: marks this walk's fetched media seen in its ledger.
    # Phase 3 calls it AFTER the import loop, never before — a file marked seen
    # but not yet imported is invisible to every later walk, so a run killed in
    # between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
    # images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
    # file on disk with no ImageRecord and imports it. Takes `skip_paths`: the
    # written paths phase 3 did not import, which stay unmarked for the same
    # reason. None on gallery-dl.
    mark_seen_after_import: Callable[..., None] | None = None


def extract_errors_warnings(stderr: str) -> str:
    """Keep only the `[error]`/`[warning]` lines from a captured stderr blob.

    A generic download-log helper (module-level so the native-ingester result
    path can shape its logs without reaching through a GalleryDLService instance).
    """
    if not stderr:
        return ""
    kept = [
        line for line in stderr.splitlines()
        if "][error]" in line.lower() or "][warning]" in line.lower()
    ]
    return "\n".join(kept)


def truncate_log(text: str, max_bytes: int = 500_000) -> str:
    """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
    if not text:
        return text
    encoded = text.encode("utf-8")
    if len(encoded) <= max_bytes:
        return text
    half = max_bytes // 2
    head = encoded[:half].decode("utf-8", errors="ignore")
    tail = encoded[-half:].decode("utf-8", errors="ignore")
    head_lines = head.count("\n")
    tail_lines = tail.count("\n")
    total_lines = text.count("\n")
    elided = max(0, total_lines - head_lines - tail_lines)
    marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
    return head + marker + tail


def make_run_stats(
    *,
    exit_code: int = 0,
    downloaded_count: int = 0,
    skipped_count: int = 0,
    per_item_failures: int = 0,
    warning_count: int = 0,
    tier_gated_count: int = 0,
    quarantined_count: int = 0,
    dead_lettered_count: int = 0,
) -> dict:
    """The canonical `run_stats` dict shape, in ONE place.

    Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
    native ingester's per-outcome tally (ingest_core) — build through this so the
    key set can't drift between backends. Phase 3 + the Logs UI read these keys.
    """
    return {
        "exit_code": exit_code,
        "downloaded_count": downloaded_count,
        "skipped_count": skipped_count,
        "per_item_failures": per_item_failures,
        "warning_count": warning_count,
        "tier_gated_count": tier_gated_count,
        "quarantined_count": quarantined_count,
        "dead_lettered_count": dead_lettered_count,
    }


# --- tier-gated classification, shared by BOTH backends ---------------------
#
# These three live together because the native ingester and the gallery-dl
# subprocess must reach the same verdict from the same number. They did not:
# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
# threw the count away, so the platforms FC owns reported a paywalled creator as
# a silent one (#874 follow-up). One predicate, spread into both, rather than
# the condition re-derived per backend.


def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
    """TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.

    Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
    posts we cannot see is tier-limited even in a week we did get their cheaper
    ones — the fact the operator needs ("there is content here you are not
    paying for") is true either way. gallery-dl has classified it this way since
    the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
    native path now matches rather than inventing a stricter rule.

    Callers must apply this only AFTER the real error categories (auth, rate
    limit, drift, …) have had their turn; tier-gating is the weakest signal and
    must never mask a genuine failure.
    """
    return ErrorType.TIER_LIMITED if tier_gated_count else None


def tier_gated_message(count: int) -> str:
    """The one wording for the tier-gated verdict, so the two backends can't
    describe the same state differently in the Logs UI."""
    return (
        f"Subscription tier does not grant access to "
        f"{count} post{'s' if count != 1 else ''}"
    )


# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
# otherwise-successful run, so without an exemption the chip is wiped the moment
# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
# render. Informational classes are the exemption — they describe the source,
# not a failure of the run.
INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})


def is_informational(error_type) -> bool:
    """True for a class that reports a state rather than a failure. Accepts an
    ErrorType or the plain string persisted on Source.error_type."""
    return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES


def walk_completed(result: DownloadResult) -> bool:
    """Did this walk reach the bottom cleanly?

    The backfill lifecycle's completion test. An informational error_type still
    counts as complete: a fully-paywalled creator's backfill DID finish, and
    treating it as unfinished re-walks the same wall until the stall counter
    trips — the creator we can see least becoming the one we fetch most.
    """
    return (
        result.success
        and result.return_code == 0
        and (result.error_type is None or is_informational(result.error_type))
    )