Factor the native-ingest orchestration out of PatreonIngester into a reusable ingest_core.Ingester base, parametrized by client/downloader/ledger-models/ constraints/key/platform/error_base. PatreonIngester becomes a thin adapter: it resolves the Patreon client/downloader, wires the seen/dead-letter models + UNIQUE-constraint names + _ledger_key into super().__init__, and overrides _failure_result with the Patreon exception taxonomy. Behavior-preserving — no table rename, no migration; the public surface (PatreonIngester, _ledger_key, DEAD_LETTER_THRESHOLD, verify_patreon_credential) is unchanged. This is the strategic seam: SubscribeStar/etc. now migrate by writing a ~40-line adapter, not by re-implementing the tick/backfill/recovery walk, tiered skip, checkpoint, and dead-letter logic. run() moved to ingest_core, so the budget test's monotonic patch repoints to ingest_core.time. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,459 @@
|
||||
"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705).
|
||||
|
||||
The orchestration that drives a native subscription walk — page a feed →
|
||||
extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download →
|
||||
mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped
|
||||
`DownloadResult`, across tick/backfill/recovery modes — is identical for every
|
||||
platform. Only four things are platform-specific, and they're INJECTED at
|
||||
construction by a thin adapter (e.g. `PatreonIngester`):
|
||||
|
||||
- `client` — `.iter_posts(feed_id, cursor)` yielding `(post, included,
|
||||
page_cursor)` + `.extract_media(post, included) -> [media]`.
|
||||
- `downloader`— `.download_post(post, media, artist_slug, is_seen) ->
|
||||
[MediaOutcome]` (status in downloaded/skipped_seen/skipped_disk
|
||||
/quarantined/error; `.path`/`.error`/`.post_id`).
|
||||
- ledger — `seen_model` + `failed_model` SQLAlchemy models (+ their
|
||||
on-conflict UNIQUE constraint names) and a `ledger_key(media)`.
|
||||
- failure map — the adapter overrides `_failure_result` (platform exception
|
||||
→ DownloadResult.error_type) and supplies `error_base` (the
|
||||
exception type the walk catches) + `platform` (result label).
|
||||
|
||||
Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker —
|
||||
never held across a network fetch ([[db-connection-held-across-subprocess]]).
|
||||
Plain-HTTP homelab: no secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or
|
||||
# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of
|
||||
# per-file HEADs. Headroom against paywalled/undownloadable items interleaving.
|
||||
_TICK_SEEN_THRESHOLD = 20
|
||||
|
||||
# plan #705 #7: after this many failed download/validate attempts a media is
|
||||
# "dead-lettered" and skipped on routine tick/backfill walks (recovery still
|
||||
# re-attempts it). Stops a permanently-broken media re-erroring forever.
|
||||
DEAD_LETTER_THRESHOLD = 3
|
||||
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
|
||||
_ERROR_MAX = 1000
|
||||
|
||||
|
||||
class Ingester:
|
||||
"""Generic native-ingest orchestration. Subclass with a platform adapter
|
||||
(see the module docstring) — or construct directly with the keyword seams."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client,
|
||||
downloader,
|
||||
session_factory: Callable[[], object],
|
||||
seen_model,
|
||||
failed_model,
|
||||
seen_constraint: str,
|
||||
failed_constraint: str,
|
||||
ledger_key: Callable[[object], str],
|
||||
platform: str,
|
||||
error_base: type[Exception],
|
||||
):
|
||||
self.client = client
|
||||
self.downloader = downloader
|
||||
self.session_factory = session_factory
|
||||
self._seen_model = seen_model
|
||||
self._failed_model = failed_model
|
||||
self._seen_constraint = seen_constraint
|
||||
self._failed_constraint = failed_constraint
|
||||
self._ledger_key = ledger_key
|
||||
self._platform = platform
|
||||
self._error_base = error_base
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
source_id: int,
|
||||
campaign_id: str,
|
||||
artist_slug: str,
|
||||
url: str,
|
||||
mode: str,
|
||||
resume_cursor: str | None = None,
|
||||
time_budget_seconds: float = 870.0,
|
||||
seen_threshold: int = _TICK_SEEN_THRESHOLD,
|
||||
) -> DownloadResult:
|
||||
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
||||
|
||||
`mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
|
||||
seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
|
||||
files). The walk stops on:
|
||||
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
|
||||
- tick early-out (seen_threshold contiguous seen) → success
|
||||
- reaching the bottom of the feed → success (rc 0)
|
||||
A client-level failure (drift / auth / network) fails the whole run loud.
|
||||
"""
|
||||
bypass_seen = mode == "recovery"
|
||||
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
|
||||
# tick has no resumable backfill state.
|
||||
checkpoint = mode in ("backfill", "recovery")
|
||||
ledger_key = self._ledger_key
|
||||
start = time.monotonic()
|
||||
log_lines: list[str] = []
|
||||
written: list[str] = []
|
||||
quarantined_paths: list[str] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
dead_lettered = 0
|
||||
skipped_count = 0
|
||||
posts_processed = 0
|
||||
consecutive_seen = 0
|
||||
emitted_cursor: str | None = None
|
||||
reached_bottom = False
|
||||
budget_hit = False
|
||||
early_out = False
|
||||
|
||||
def _result(
|
||||
*, success: bool, return_code: int,
|
||||
error_type: ErrorType | None, error_message: str | None,
|
||||
) -> DownloadResult:
|
||||
# plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor
|
||||
# directly instead of regex-scraping a reconstructed stdout. stdout
|
||||
# stays a human-readable summary (no fake `Cursor:` lines).
|
||||
return DownloadResult(
|
||||
success=success,
|
||||
url=url,
|
||||
artist_slug=artist_slug,
|
||||
platform=self._platform,
|
||||
files_downloaded=downloaded,
|
||||
files_quarantined=quarantined,
|
||||
quarantined_paths=list(quarantined_paths),
|
||||
written_paths=written,
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
error_type=error_type,
|
||||
error_message=error_message,
|
||||
duration_seconds=time.monotonic() - start,
|
||||
cursor=emitted_cursor,
|
||||
posts_processed=posts_processed,
|
||||
run_stats={
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": downloaded,
|
||||
"skipped_count": skipped_count,
|
||||
"per_item_failures": errors,
|
||||
"warning_count": 0,
|
||||
"tier_gated_count": 0,
|
||||
"quarantined_count": quarantined,
|
||||
"dead_lettered_count": dead_lettered,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
for post, included, page_cursor in self.client.iter_posts(
|
||||
campaign_id, cursor=resume_cursor
|
||||
):
|
||||
# Checkpoint the cursor that FETCHED this page the moment we
|
||||
# START it — so a chunk cut mid-page resumes the page, not the one
|
||||
# after it. Carried as DownloadResult.cursor (plan #704).
|
||||
if page_cursor and page_cursor != emitted_cursor:
|
||||
emitted_cursor = page_cursor
|
||||
# plan #705 #6: persist the cursor at each page boundary so a
|
||||
# worker SIGKILL mid-chunk resumes near the crash, not the
|
||||
# chunk start. (phase 3 still writes the final cursor — same
|
||||
# value; this is the crash-safety net.)
|
||||
if checkpoint:
|
||||
self._checkpoint_cursor(source_id, emitted_cursor)
|
||||
|
||||
# Time-box check at the post boundary (coarse, like a gallery-dl
|
||||
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
||||
if time.monotonic() - start >= time_budget_seconds:
|
||||
budget_hit = True
|
||||
break
|
||||
|
||||
posts_processed += 1
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
continue
|
||||
|
||||
keys = [ledger_key(m) for m in media]
|
||||
# Recovery bypasses BOTH the seen-ledger AND the dead-letter
|
||||
# ledger (the operator's "try everything again"); routine walks
|
||||
# skip seen + dead media (tier-1 + tier-1.5, plan #705 #7).
|
||||
dead = set() if bypass_seen else self._dead_keys(source_id, keys)
|
||||
seen = (
|
||||
set()
|
||||
if bypass_seen
|
||||
else self._seen_keys(source_id, keys)
|
||||
)
|
||||
skip = seen | dead
|
||||
|
||||
def _is_skip(m, _skip=skip) -> bool:
|
||||
return ledger_key(m) in _skip
|
||||
|
||||
outcomes = self.downloader.download_post(
|
||||
post, media, artist_slug, is_seen=_is_skip
|
||||
)
|
||||
|
||||
to_mark: list[tuple[str, str]] = []
|
||||
to_clear: list[str] = [] # recovered → drop any dead-letter row
|
||||
to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error)
|
||||
for media_item, outcome in zip(media, outcomes, strict=False):
|
||||
key = ledger_key(media_item)
|
||||
if key in dead:
|
||||
dead_lettered += 1 # skipped because previously dead
|
||||
if outcome.status == "downloaded":
|
||||
downloaded += 1
|
||||
if outcome.path is not None:
|
||||
written.append(str(outcome.path))
|
||||
to_mark.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
consecutive_seen = 0
|
||||
elif outcome.status == "skipped_disk":
|
||||
# Already on disk (a prior run). Reconcile the ledger so a
|
||||
# later tick skips it at tier-1 without a disk stat, but
|
||||
# do NOT re-feed it to phase 3 — attach_in_place would see
|
||||
# the duplicate sha256 and unlink the on-disk copy.
|
||||
to_mark.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
elif outcome.status == "skipped_seen":
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
elif outcome.status == "quarantined":
|
||||
# New content that failed validation (corrupt) — counted
|
||||
# distinctly so the run surfaces a real quarantined total.
|
||||
# Not marked seen (a later walk may re-fetch a fixed file);
|
||||
# it IS new content, so it breaks the run-of-seen. Counts
|
||||
# toward the dead-letter ledger (plan #705 #7).
|
||||
quarantined += 1
|
||||
if outcome.path is not None:
|
||||
quarantined_paths.append(str(outcome.path))
|
||||
to_fail.append((key, media_item.post_id, outcome.error or "quarantined"))
|
||||
consecutive_seen = 0
|
||||
elif outcome.status == "error":
|
||||
errors += 1
|
||||
to_fail.append((key, media_item.post_id, outcome.error or "error"))
|
||||
# An error neither advances nor resets the run-of-seen.
|
||||
|
||||
if mode == "tick" and consecutive_seen >= seen_threshold:
|
||||
early_out = True
|
||||
break
|
||||
|
||||
# Persist ledger changes AFTER the network fetch, on short
|
||||
# sessions: mark downloaded/on-disk seen, clear any dead-letter
|
||||
# for recovered media, and record failures (plan #705 #7).
|
||||
if to_mark:
|
||||
self._mark_seen(source_id, to_mark)
|
||||
if to_clear:
|
||||
self._clear_failures(source_id, to_clear)
|
||||
if to_fail:
|
||||
self._record_failures(source_id, to_fail)
|
||||
|
||||
if early_out:
|
||||
break
|
||||
else:
|
||||
reached_bottom = True
|
||||
except self._error_base as exc:
|
||||
# The platform's client-error base — _failure_result (adapter)
|
||||
# maps it to a typed error.
|
||||
return self._failure_result(exc, _result)
|
||||
|
||||
if errors:
|
||||
log_lines.append(f"{errors} media item(s) failed")
|
||||
if quarantined:
|
||||
log_lines.append(f"{quarantined} media item(s) quarantined (invalid)")
|
||||
if dead_lettered:
|
||||
log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)")
|
||||
log_lines.append(
|
||||
f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
|
||||
f"{skipped_count} skipped, {quarantined} quarantined, "
|
||||
f"{dead_lettered} dead-lettered, {errors} error(s), "
|
||||
f"{posts_processed} post(s)"
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
|
||||
if budget_hit:
|
||||
# A chunk that hit its time-box but made forward progress is a
|
||||
# NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the
|
||||
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
||||
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
||||
# subprocess TimeoutExpired so completion detection stays false.
|
||||
made_progress = downloaded > 0 or emitted_cursor != resume_cursor
|
||||
if made_progress:
|
||||
return _result(
|
||||
success=False, return_code=-1,
|
||||
error_type=ErrorType.PARTIAL,
|
||||
error_message=(
|
||||
f"Backfill chunk: {downloaded} file(s) — continuing"
|
||||
),
|
||||
)
|
||||
return _result(
|
||||
success=False, return_code=-1,
|
||||
error_type=ErrorType.TIMEOUT,
|
||||
error_message="Chunk timed out with no progress",
|
||||
)
|
||||
|
||||
# Normal success: reached the bottom, or a tick that early-outed. rc 0 +
|
||||
# error_type None is REQUIRED for a backfill/recovery walk that reached
|
||||
# the bottom to be marked COMPLETE by
|
||||
# download_service._apply_backfill_lifecycle — so we return None even
|
||||
# when downloaded == 0 (a re-confirming walk that found nothing new still
|
||||
# completed). success=True maps to status "ok" regardless. A tick that
|
||||
# early-outed also returns here; ticks never set backfill state so the
|
||||
# lifecycle is a no-op for them.
|
||||
return _result(
|
||||
success=True, return_code=0,
|
||||
error_type=None, error_message=None,
|
||||
)
|
||||
|
||||
# -- failure mapping (adapter overrides) -------------------------------
|
||||
|
||||
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
||||
"""Map a platform client-error to a typed failed DownloadResult. The base
|
||||
gives a safe default; adapters override with their exception taxonomy."""
|
||||
log.warning("%s ingest failed: %s", self._platform, exc)
|
||||
return _result(
|
||||
success=False, return_code=1,
|
||||
error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
|
||||
)
|
||||
|
||||
# -- seen-ledger (short-lived sessions) --------------------------------
|
||||
|
||||
def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]:
|
||||
"""Which of `keys` are already in the seen-ledger for this source.
|
||||
|
||||
One short SELECT on its own session — opened and closed without any
|
||||
network in between (the GETs happen after, in download_post).
|
||||
"""
|
||||
if not keys:
|
||||
return set()
|
||||
with self.session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(self._seen_model.filehash).where(
|
||||
self._seen_model.source_id == source_id,
|
||||
self._seen_model.filehash.in_(keys),
|
||||
)
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
def _checkpoint_cursor(self, source_id: int, cursor: str) -> None:
|
||||
"""Persist the in-progress backfill cursor mid-walk (plan #705 #6).
|
||||
|
||||
ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just
|
||||
`_backfill_cursor`, cast back — so it never clobbers operator config or
|
||||
the other backfill keys (no read-modify-write race). The in-flight guard
|
||||
means only this source's one download runs at a time; a concurrent
|
||||
operator stop is benign (a stray cursor with no `_backfill_state` is
|
||||
ignored by tick mode and cleared on the next start).
|
||||
"""
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
text(
|
||||
"UPDATE source SET config_overrides = jsonb_set("
|
||||
" coalesce(config_overrides::jsonb, '{}'::jsonb),"
|
||||
" '{_backfill_cursor}', to_jsonb(cast(:cur AS text))"
|
||||
")::json WHERE id = :sid"
|
||||
),
|
||||
{"cur": cursor, "sid": source_id},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
|
||||
"""Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
|
||||
|
||||
ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a
|
||||
re-sighting — or a concurrent walk — is a harmless no-op
|
||||
([[scalar_one_or_none-duplicates]]: never check-then-insert without the
|
||||
DB constraint backing it). De-dup the batch locally first so a single
|
||||
page can't present the same key twice to one INSERT.
|
||||
"""
|
||||
seen_local: set[str] = set()
|
||||
values = []
|
||||
for key, post_id in items:
|
||||
if key in seen_local:
|
||||
continue
|
||||
seen_local.add(key)
|
||||
values.append(
|
||||
{"source_id": source_id, "filehash": key, "post_id": post_id}
|
||||
)
|
||||
if not values:
|
||||
return
|
||||
with self.session_factory() as session:
|
||||
stmt = pg_insert(self._seen_model).values(values)
|
||||
stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint)
|
||||
session.execute(stmt)
|
||||
session.commit()
|
||||
|
||||
# -- dead-letter ledger (plan #705 #7) ---------------------------------
|
||||
|
||||
def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]:
|
||||
"""Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead).
|
||||
One short SELECT; recovery never calls this (it re-attempts dead media)."""
|
||||
if not keys:
|
||||
return set()
|
||||
with self.session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(self._failed_model.filehash).where(
|
||||
self._failed_model.source_id == source_id,
|
||||
self._failed_model.filehash.in_(keys),
|
||||
self._failed_model.attempts >= DEAD_LETTER_THRESHOLD,
|
||||
)
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
def _record_failures(
|
||||
self, source_id: int, items: list[tuple[str, str, str]]
|
||||
) -> None:
|
||||
"""Upsert-increment the dead-letter ledger for failed media. On conflict
|
||||
bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the
|
||||
upsert — no check-then-insert). De-dup the batch (one row/key, last error
|
||||
wins)."""
|
||||
by_key: dict[str, str] = {}
|
||||
for key, _post_id, err in items:
|
||||
by_key[key] = (err or "")[:_ERROR_MAX]
|
||||
if not by_key:
|
||||
return
|
||||
values = [
|
||||
{"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e}
|
||||
for k, e in by_key.items()
|
||||
]
|
||||
with self.session_factory() as session:
|
||||
stmt = pg_insert(self._failed_model).values(values)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
constraint=self._failed_constraint,
|
||||
set_={
|
||||
"attempts": self._failed_model.attempts + 1,
|
||||
"last_error": stmt.excluded.last_error,
|
||||
"last_failed_at": func.now(),
|
||||
},
|
||||
)
|
||||
session.execute(stmt)
|
||||
session.commit()
|
||||
|
||||
def _clear_failures(self, source_id: int, keys: list[str]) -> None:
|
||||
"""Drop dead-letter rows for media that just downloaded cleanly — they
|
||||
recovered. A no-op DELETE for keys that were never failing."""
|
||||
unique = list(dict.fromkeys(keys))
|
||||
if not unique:
|
||||
return
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
delete(self._failed_model).where(
|
||||
self._failed_model.source_id == source_id,
|
||||
self._failed_model.filehash.in_(unique),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
Reference in New Issue
Block a user