bvandeusen
cd43439401
Owning the walk lets Stop interrupt a live backfill chunk instead of letting it run to its ~14.5-min time-box. ingest_core.run now polls _backfill_state at each page boundary (a short SELECT, never held across the walk) and bails with PARTIAL when an operator Stop has popped it. Latched on the first observed "running" state so a run invoked without one (unit test / stale call) never self-cancels. Progress is already checkpointed per-page, so a restart resumes from the cursor; Stop clears it for a clean reset. No UI change — the existing Stop button now just takes effect immediately. Tests: _still_running reads the state; a latched run bails PARTIAL at the next boundary when the state disappears (only the pre-cancel post ran). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
540 lines
25 KiB
Python
540 lines
25 KiB
Python
"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705).
|
|
|
|
The orchestration that drives a native subscription walk — page a feed →
|
|
extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download →
|
|
mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped
|
|
`DownloadResult`, across tick/backfill/recovery modes — is identical for every
|
|
platform. Only four things are platform-specific, and they're INJECTED at
|
|
construction by a thin adapter (e.g. `PatreonIngester`):
|
|
|
|
- `client` — `.iter_posts(feed_id, cursor)` yielding `(post, included,
|
|
page_cursor)` + `.extract_media(post, included) -> [media]`.
|
|
- `downloader`— `.download_post(post, media, artist_slug, is_seen) ->
|
|
[MediaOutcome]` (status in downloaded/skipped_seen/skipped_disk
|
|
/quarantined/error; `.path`/`.error`/`.post_id`).
|
|
- ledger — `seen_model` + `failed_model` SQLAlchemy models (+ their
|
|
on-conflict UNIQUE constraint names) and a `ledger_key(media)`.
|
|
- failure map — the adapter overrides `_failure_result` (platform exception
|
|
→ DownloadResult.error_type) and supplies `error_base` (the
|
|
exception type the walk catches) + `platform` (result label).
|
|
|
|
Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker —
|
|
never held across a network fetch ([[db-connection-held-across-subprocess]]).
|
|
Plain-HTTP homelab: no secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from collections.abc import Callable
|
|
|
|
from sqlalchemy import delete, func, select, text
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
from .gallery_dl import DownloadResult, ErrorType, make_run_stats
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or
|
|
# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of
|
|
# per-file HEADs. Headroom against paywalled/undownloadable items interleaving.
|
|
_TICK_SEEN_THRESHOLD = 20
|
|
|
|
# plan #705 #7: after this many failed download/validate attempts a media is
|
|
# "dead-lettered" and skipped on routine tick/backfill walks (recovery still
|
|
# re-attempts it). Stops a permanently-broken media re-erroring forever.
|
|
DEAD_LETTER_THRESHOLD = 3
|
|
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
|
|
_ERROR_MAX = 1000
|
|
|
|
|
|
class Ingester:
|
|
"""Generic native-ingest orchestration. Subclass with a platform adapter
|
|
(see the module docstring) — or construct directly with the keyword seams."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
client,
|
|
downloader,
|
|
session_factory: Callable[[], object],
|
|
seen_model,
|
|
failed_model,
|
|
seen_constraint: str,
|
|
failed_constraint: str,
|
|
ledger_key: Callable[[object], str],
|
|
platform: str,
|
|
error_base: type[Exception],
|
|
):
|
|
self.client = client
|
|
self.downloader = downloader
|
|
self.session_factory = session_factory
|
|
self._seen_model = seen_model
|
|
self._failed_model = failed_model
|
|
self._seen_constraint = seen_constraint
|
|
self._failed_constraint = failed_constraint
|
|
self._ledger_key = ledger_key
|
|
self._platform = platform
|
|
self._error_base = error_base
|
|
|
|
# -- public ------------------------------------------------------------
|
|
|
|
def run(
|
|
self,
|
|
*,
|
|
source_id: int,
|
|
campaign_id: str,
|
|
artist_slug: str,
|
|
url: str,
|
|
mode: str,
|
|
resume_cursor: str | None = None,
|
|
time_budget_seconds: float = 870.0,
|
|
seen_threshold: int = _TICK_SEEN_THRESHOLD,
|
|
posts_base: int = 0,
|
|
) -> DownloadResult:
|
|
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
|
|
|
`mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
|
|
seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
|
|
files). The walk stops on:
|
|
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
|
|
- tick early-out (seen_threshold contiguous seen) → success
|
|
- reaching the bottom of the feed → success (rc 0)
|
|
A client-level failure (drift / auth / network) fails the whole run loud.
|
|
"""
|
|
bypass_seen = mode == "recovery"
|
|
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
|
|
# tick has no resumable backfill state.
|
|
checkpoint = mode in ("backfill", "recovery")
|
|
ledger_key = self._ledger_key
|
|
start = time.monotonic()
|
|
log_lines: list[str] = []
|
|
written: list[str] = []
|
|
quarantined_paths: list[str] = []
|
|
downloaded = 0
|
|
errors = 0
|
|
quarantined = 0
|
|
dead_lettered = 0
|
|
skipped_count = 0
|
|
posts_processed = 0
|
|
# Net-new posts THIS chunk for the live progress badge (plan #704 #5);
|
|
# excludes the re-walked resume page so _backfill_posts stays a monotonic
|
|
# absolute across chunks instead of an inflating sum. posts_processed
|
|
# stays the gross per-chunk count used for the run summary.
|
|
chunk_new_posts = 0
|
|
consecutive_seen = 0
|
|
emitted_cursor: str | None = None
|
|
reached_bottom = False
|
|
budget_hit = False
|
|
early_out = False
|
|
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
|
cancel_armed = False # latched once we observe a live "running" state
|
|
|
|
def _result(
|
|
*, success: bool, return_code: int,
|
|
error_type: ErrorType | None, error_message: str | None,
|
|
) -> DownloadResult:
|
|
# plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor
|
|
# directly instead of regex-scraping a reconstructed stdout. stdout
|
|
# stays a human-readable summary (no fake `Cursor:` lines).
|
|
return DownloadResult(
|
|
success=success,
|
|
url=url,
|
|
artist_slug=artist_slug,
|
|
platform=self._platform,
|
|
files_downloaded=downloaded,
|
|
files_quarantined=quarantined,
|
|
quarantined_paths=list(quarantined_paths),
|
|
written_paths=written,
|
|
stdout="\n".join(log_lines),
|
|
stderr="",
|
|
return_code=return_code,
|
|
error_type=error_type,
|
|
error_message=error_message,
|
|
duration_seconds=time.monotonic() - start,
|
|
cursor=emitted_cursor,
|
|
posts_processed=posts_processed,
|
|
run_stats=make_run_stats(
|
|
exit_code=return_code,
|
|
downloaded_count=downloaded,
|
|
skipped_count=skipped_count,
|
|
per_item_failures=errors,
|
|
quarantined_count=quarantined,
|
|
dead_lettered_count=dead_lettered,
|
|
),
|
|
)
|
|
|
|
try:
|
|
for post, included, page_cursor in self.client.iter_posts(
|
|
campaign_id, cursor=resume_cursor
|
|
):
|
|
# Checkpoint the cursor that FETCHED this page the moment we
|
|
# START it — so a chunk cut mid-page resumes the page, not the one
|
|
# after it. Carried as DownloadResult.cursor (plan #704).
|
|
if page_cursor and page_cursor != emitted_cursor:
|
|
emitted_cursor = page_cursor
|
|
# plan #705 #6: persist the cursor at each page boundary so a
|
|
# worker SIGKILL mid-chunk resumes near the crash, not the
|
|
# chunk start. (phase 3 still writes the final cursor — same
|
|
# value; this is the crash-safety net.) plan #704 #5: persist
|
|
# the live posts count alongside it so the badge climbs DURING
|
|
# the chunk, not only when it ends.
|
|
if checkpoint:
|
|
# plan #708 B4: an operator Stop pops `_backfill_state` —
|
|
# bail at the page boundary (progress already checkpointed)
|
|
# before more network work, so the live chunk halts
|
|
# promptly instead of running to its time-box. LATCH on the
|
|
# first observed "running" state, so a run invoked WITHOUT a
|
|
# running state (a unit test, or a stale call) never
|
|
# spuriously self-cancels. A short SELECT, never held.
|
|
if self._still_running(source_id):
|
|
cancel_armed = True
|
|
elif cancel_armed:
|
|
stopped = True
|
|
break
|
|
self._checkpoint_cursor(source_id, emitted_cursor)
|
|
self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
|
|
|
|
# Time-box check at the post boundary (coarse, like a gallery-dl
|
|
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
|
if time.monotonic() - start >= time_budget_seconds:
|
|
budget_hit = True
|
|
break
|
|
|
|
posts_processed += 1
|
|
# The resume page (its cursor == resume_cursor) was already
|
|
# counted by the chunk that checkpointed it — don't re-count it
|
|
# into the persisted badge (plan #704 #5). First chunk has
|
|
# resume_cursor None, so everything counts.
|
|
if not (resume_cursor and page_cursor == resume_cursor):
|
|
chunk_new_posts += 1
|
|
media = self.client.extract_media(post, included)
|
|
if not media:
|
|
continue
|
|
|
|
keys = [ledger_key(m) for m in media]
|
|
# Recovery bypasses BOTH the seen-ledger AND the dead-letter
|
|
# ledger (the operator's "try everything again"); routine walks
|
|
# skip seen + dead media (tier-1 + tier-1.5, plan #705 #7).
|
|
dead = set() if bypass_seen else self._dead_keys(source_id, keys)
|
|
seen = (
|
|
set()
|
|
if bypass_seen
|
|
else self._seen_keys(source_id, keys)
|
|
)
|
|
skip = seen | dead
|
|
|
|
def _is_skip(m, _skip=skip) -> bool:
|
|
return ledger_key(m) in _skip
|
|
|
|
outcomes = self.downloader.download_post(
|
|
post, media, artist_slug, is_seen=_is_skip
|
|
)
|
|
|
|
to_mark: list[tuple[str, str]] = []
|
|
to_clear: list[str] = [] # recovered → drop any dead-letter row
|
|
to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error)
|
|
for media_item, outcome in zip(media, outcomes, strict=False):
|
|
key = ledger_key(media_item)
|
|
if key in dead:
|
|
dead_lettered += 1 # skipped because previously dead
|
|
if outcome.status == "downloaded":
|
|
downloaded += 1
|
|
if outcome.path is not None:
|
|
written.append(str(outcome.path))
|
|
to_mark.append((key, media_item.post_id))
|
|
to_clear.append(key)
|
|
consecutive_seen = 0
|
|
elif outcome.status == "skipped_disk":
|
|
# Already on disk (a prior run). Reconcile the ledger so a
|
|
# later tick skips it at tier-1 without a disk stat, but
|
|
# do NOT re-feed it to phase 3 — attach_in_place would see
|
|
# the duplicate sha256 and unlink the on-disk copy.
|
|
to_mark.append((key, media_item.post_id))
|
|
to_clear.append(key)
|
|
skipped_count += 1
|
|
consecutive_seen += 1
|
|
elif outcome.status == "skipped_seen":
|
|
skipped_count += 1
|
|
consecutive_seen += 1
|
|
elif outcome.status == "quarantined":
|
|
# New content that failed validation (corrupt) — counted
|
|
# distinctly so the run surfaces a real quarantined total.
|
|
# Not marked seen (a later walk may re-fetch a fixed file);
|
|
# it IS new content, so it breaks the run-of-seen. Counts
|
|
# toward the dead-letter ledger (plan #705 #7).
|
|
quarantined += 1
|
|
if outcome.path is not None:
|
|
quarantined_paths.append(str(outcome.path))
|
|
to_fail.append((key, media_item.post_id, outcome.error or "quarantined"))
|
|
consecutive_seen = 0
|
|
elif outcome.status == "error":
|
|
errors += 1
|
|
to_fail.append((key, media_item.post_id, outcome.error or "error"))
|
|
# An error neither advances nor resets the run-of-seen.
|
|
|
|
if mode == "tick" and consecutive_seen >= seen_threshold:
|
|
early_out = True
|
|
break
|
|
|
|
# Persist ledger changes AFTER the network fetch, on short
|
|
# sessions: mark downloaded/on-disk seen, clear any dead-letter
|
|
# for recovered media, and record failures (plan #705 #7).
|
|
if to_mark:
|
|
self._mark_seen(source_id, to_mark)
|
|
if to_clear:
|
|
self._clear_failures(source_id, to_clear)
|
|
if to_fail:
|
|
self._record_failures(source_id, to_fail)
|
|
|
|
if early_out:
|
|
break
|
|
else:
|
|
reached_bottom = True
|
|
except self._error_base as exc:
|
|
# The platform's client-error base — _failure_result (adapter)
|
|
# maps it to a typed error.
|
|
return self._failure_result(exc, _result)
|
|
|
|
# plan #708 B4: a Stop already popped the backfill state (incl. cursor +
|
|
# posts), so don't re-write them — return PARTIAL (reads as "ok/progress",
|
|
# the lifecycle no-ops since state is gone) instead of a false "complete".
|
|
if stopped:
|
|
return _result(
|
|
success=False, return_code=-1,
|
|
error_type=ErrorType.PARTIAL,
|
|
error_message=f"Stopped by operator: {downloaded} file(s) this chunk",
|
|
)
|
|
|
|
# Final authoritative posts count for the badge — captures the last page
|
|
# after the last boundary write and the time-box break (plan #704 #5).
|
|
if checkpoint:
|
|
self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
|
|
|
|
if errors:
|
|
log_lines.append(f"{errors} media item(s) failed")
|
|
if quarantined:
|
|
log_lines.append(f"{quarantined} media item(s) quarantined (invalid)")
|
|
if dead_lettered:
|
|
log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)")
|
|
log_lines.append(
|
|
f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
|
|
f"{skipped_count} skipped, {quarantined} quarantined, "
|
|
f"{dead_lettered} dead-lettered, {errors} error(s), "
|
|
f"{posts_processed} post(s)"
|
|
+ (", reached end" if reached_bottom else "")
|
|
+ (", time-boxed" if budget_hit else "")
|
|
)
|
|
|
|
if budget_hit:
|
|
# A chunk that hit its time-box but made forward progress is a
|
|
# NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the
|
|
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
|
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
|
# subprocess TimeoutExpired so completion detection stays false.
|
|
made_progress = downloaded > 0 or emitted_cursor != resume_cursor
|
|
if made_progress:
|
|
return _result(
|
|
success=False, return_code=-1,
|
|
error_type=ErrorType.PARTIAL,
|
|
error_message=(
|
|
f"Backfill chunk: {downloaded} file(s) — continuing"
|
|
),
|
|
)
|
|
return _result(
|
|
success=False, return_code=-1,
|
|
error_type=ErrorType.TIMEOUT,
|
|
error_message="Chunk timed out with no progress",
|
|
)
|
|
|
|
# Normal success: reached the bottom, or a tick that early-outed. rc 0 +
|
|
# error_type None is REQUIRED for a backfill/recovery walk that reached
|
|
# the bottom to be marked COMPLETE by
|
|
# download_service._apply_backfill_lifecycle — so we return None even
|
|
# when downloaded == 0 (a re-confirming walk that found nothing new still
|
|
# completed). success=True maps to status "ok" regardless. A tick that
|
|
# early-outed also returns here; ticks never set backfill state so the
|
|
# lifecycle is a no-op for them.
|
|
return _result(
|
|
success=True, return_code=0,
|
|
error_type=None, error_message=None,
|
|
)
|
|
|
|
# -- failure mapping (adapter overrides) -------------------------------
|
|
|
|
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
|
"""Map a platform client-error to a typed failed DownloadResult. The base
|
|
gives a safe default; adapters override with their exception taxonomy."""
|
|
log.warning("%s ingest failed: %s", self._platform, exc)
|
|
return _result(
|
|
success=False, return_code=1,
|
|
error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
|
|
)
|
|
|
|
# -- seen-ledger (short-lived sessions) --------------------------------
|
|
|
|
def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]:
|
|
"""Which of `keys` are already in the seen-ledger for this source.
|
|
|
|
One short SELECT on its own session — opened and closed without any
|
|
network in between (the GETs happen after, in download_post).
|
|
"""
|
|
if not keys:
|
|
return set()
|
|
with self.session_factory() as session:
|
|
rows = session.execute(
|
|
select(self._seen_model.filehash).where(
|
|
self._seen_model.source_id == source_id,
|
|
self._seen_model.filehash.in_(keys),
|
|
)
|
|
).scalars().all()
|
|
return set(rows)
|
|
|
|
def _checkpoint_cursor(self, source_id: int, cursor: str) -> None:
|
|
"""Persist the in-progress backfill cursor mid-walk (plan #705 #6).
|
|
|
|
ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just
|
|
`_backfill_cursor`, cast back — so it never clobbers operator config or
|
|
the other backfill keys (no read-modify-write race). The in-flight guard
|
|
means only this source's one download runs at a time; a concurrent
|
|
operator stop is benign (a stray cursor with no `_backfill_state` is
|
|
ignored by tick mode and cleared on the next start).
|
|
"""
|
|
with self.session_factory() as session:
|
|
session.execute(
|
|
text(
|
|
"UPDATE source SET config_overrides = jsonb_set("
|
|
" coalesce(config_overrides::jsonb, '{}'::jsonb),"
|
|
" '{_backfill_cursor}', to_jsonb(cast(:cur AS text))"
|
|
")::json WHERE id = :sid"
|
|
),
|
|
{"cur": cursor, "sid": source_id},
|
|
)
|
|
session.commit()
|
|
|
|
def _still_running(self, source_id: int) -> bool:
|
|
"""True while the source is armed for a deep walk (plan #708 B4).
|
|
|
|
An operator Stop (`source_service.stop_backfill`) pops `_backfill_state`,
|
|
so a False here means "cancel this chunk now". One short SELECT on its own
|
|
session — never held across the walk
|
|
([[db-connection-held-across-subprocess]])."""
|
|
with self.session_factory() as session:
|
|
state = session.execute(
|
|
text(
|
|
"SELECT config_overrides::jsonb ->> '_backfill_state' "
|
|
"FROM source WHERE id = :sid"
|
|
),
|
|
{"sid": source_id},
|
|
).scalar_one_or_none()
|
|
return state == "running"
|
|
|
|
def _checkpoint_posts(self, source_id: int, posts: int) -> None:
|
|
"""Persist the live backfill posts-processed count mid-walk (plan #704 #5).
|
|
|
|
Same atomic single-key jsonb_set dance as _checkpoint_cursor, on the
|
|
`_backfill_posts` key (cast to a JSON number) — so the progress badge
|
|
climbs DURING a chunk without clobbering operator config or the cursor.
|
|
The ingester OWNS this key now; download_service no longer accumulates it
|
|
post-chunk (which lagged a whole chunk and over-counted the resume page).
|
|
"""
|
|
with self.session_factory() as session:
|
|
session.execute(
|
|
text(
|
|
"UPDATE source SET config_overrides = jsonb_set("
|
|
" coalesce(config_overrides::jsonb, '{}'::jsonb),"
|
|
" '{_backfill_posts}', to_jsonb(cast(:posts AS int))"
|
|
")::json WHERE id = :sid"
|
|
),
|
|
{"posts": posts, "sid": source_id},
|
|
)
|
|
session.commit()
|
|
|
|
def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
|
|
"""Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
|
|
|
|
ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a
|
|
re-sighting — or a concurrent walk — is a harmless no-op
|
|
([[scalar_one_or_none-duplicates]]: never check-then-insert without the
|
|
DB constraint backing it). De-dup the batch locally first so a single
|
|
page can't present the same key twice to one INSERT.
|
|
"""
|
|
seen_local: set[str] = set()
|
|
values = []
|
|
for key, post_id in items:
|
|
if key in seen_local:
|
|
continue
|
|
seen_local.add(key)
|
|
values.append(
|
|
{"source_id": source_id, "filehash": key, "post_id": post_id}
|
|
)
|
|
if not values:
|
|
return
|
|
with self.session_factory() as session:
|
|
stmt = pg_insert(self._seen_model).values(values)
|
|
stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint)
|
|
session.execute(stmt)
|
|
session.commit()
|
|
|
|
# -- dead-letter ledger (plan #705 #7) ---------------------------------
|
|
|
|
def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]:
|
|
"""Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead).
|
|
One short SELECT; recovery never calls this (it re-attempts dead media)."""
|
|
if not keys:
|
|
return set()
|
|
with self.session_factory() as session:
|
|
rows = session.execute(
|
|
select(self._failed_model.filehash).where(
|
|
self._failed_model.source_id == source_id,
|
|
self._failed_model.filehash.in_(keys),
|
|
self._failed_model.attempts >= DEAD_LETTER_THRESHOLD,
|
|
)
|
|
).scalars().all()
|
|
return set(rows)
|
|
|
|
def _record_failures(
|
|
self, source_id: int, items: list[tuple[str, str, str]]
|
|
) -> None:
|
|
"""Upsert-increment the dead-letter ledger for failed media. On conflict
|
|
bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the
|
|
upsert — no check-then-insert). De-dup the batch (one row/key, last error
|
|
wins)."""
|
|
by_key: dict[str, str] = {}
|
|
for key, _post_id, err in items:
|
|
by_key[key] = (err or "")[:_ERROR_MAX]
|
|
if not by_key:
|
|
return
|
|
values = [
|
|
{"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e}
|
|
for k, e in by_key.items()
|
|
]
|
|
with self.session_factory() as session:
|
|
stmt = pg_insert(self._failed_model).values(values)
|
|
stmt = stmt.on_conflict_do_update(
|
|
constraint=self._failed_constraint,
|
|
set_={
|
|
"attempts": self._failed_model.attempts + 1,
|
|
"last_error": stmt.excluded.last_error,
|
|
"last_failed_at": func.now(),
|
|
},
|
|
)
|
|
session.execute(stmt)
|
|
session.commit()
|
|
|
|
def _clear_failures(self, source_id: int, keys: list[str]) -> None:
|
|
"""Drop dead-letter rows for media that just downloaded cleanly — they
|
|
recovered. A no-op DELETE for keys that were never failing."""
|
|
unique = list(dict.fromkeys(keys))
|
|
if not unique:
|
|
return
|
|
with self.session_factory() as session:
|
|
session.execute(
|
|
delete(self._failed_model).where(
|
|
self._failed_model.source_id == source_id,
|
|
self._failed_model.filehash.in_(unique),
|
|
)
|
|
)
|
|
session.commit()
|