bvandeusen
19aece1fc4
Routine subscription polls walked the entire post history every tick
even when nothing had changed, because gallery-dl's default `skip: True`
continues iterating archived posts. A creator with ~550 archived posts
(Knuxy patreon) saturates the 870s wall-clock cap before completing,
even with zero downloads needed. Plus, a tier-limited run that
downloaded hundreds of files but ran out the clock should be a
warning, not an error.
Two coupled changes, both operator-flagged 2026-06-01:
* **Tick mode (default, cron polls).** New `TICK_SKIP_VALUE = "exit:20"`
asks gallery-dl to exit after 20 contiguous archived items. Fresh
subscriptions + new-content cases still walk normally; established
subscription with zero new content exits in ~30s of HEAD requests
instead of pegging the timeout. 20 (not 5) gives headroom against
paywall warnings interleaving with archived items.
* **Backfill mode (explicit, operator-triggered).** Sticky for N runs
via new `Source.backfill_runs_remaining` (alembic 0031). While > 0,
downloads use `skip: True` + 1800s timeout. Auto-decrements per run
with early-reset to 0 when a clean run finds zero files (queue
drained). N defaults to 3 — multiple runs give the system enough
budget to finish a deep walk across timeout boundaries. New
`POST /api/sources/{id}/backfill` arms the source; "Deep scan"
button on each SourceRow (chip shows remaining count) wires it.
Plus partial-success classifier: non-zero gallery-dl exit + ≥1 file
downloaded + no source-level error fires `ErrorType.PARTIAL`, which
download_service maps to `status=\"ok\"`. The run did real work; the
next tick continues via gallery-dl's archive. No more red events for
"timed out mid-walk after downloading 300 files."
Retires `SourceConfig.skip_existing` — skip value is now derived from
the source state and passed as a separate `skip_value` parameter
through download() / _build_config_for_source(). `GD_DEFAULTS` drops
the now-dead key (was inert data after this refactor).
Tests cover:
* tick + backfill skip-value emission in _build_config_for_source
* PARTIAL classifier branch + TIER_LIMITED-wins-over-PARTIAL ordering
* SourceService.set_backfill_runs validation + persistence
* /api/sources/{id}/backfill 200/400/404 paths
* download_service auto-decrement / auto-reset / tick-mode-no-touch
* PARTIAL → status=ok in the orchestrator (no consecutive_failures bump)
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
"""Layer-2 one-shot re-download remediation for corrupt imported files.
|
|
|
|
When an import fails on a file that came from a known, pollable
|
|
subscription Source, deleting the bad copy and re-running the source's
|
|
downloader can fetch a fresh, unblemished copy. This only helps when:
|
|
|
|
- the corruption is in transit / on disk (not at the source), AND
|
|
- the file resolves to an ENABLED Source with a real feed URL
|
|
(a `sidecar:<platform>:<slug>` synthetic anchor is not pollable),
|
|
AND
|
|
- we haven't already re-fetched this task once (bounded by
|
|
ImportTask.refetched so source-side corruption can't loop).
|
|
|
|
Filesystem-only imports with no resolvable Source return 'no_source' —
|
|
the operator's only remediation there is to replace the file on disk.
|
|
|
|
Operator-requested 2026-05-28 (Layer 2).
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..models import Artist, ImportTask, Source
|
|
from ..utils.paths import derive_top_level_artist
|
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
|
from ..utils.slug import slugify
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def resolve_refetch_source(
|
|
session: Session, source_path: str, import_root: Path,
|
|
) -> Source | None:
|
|
"""Find an enabled, real-URL Source for the file's (artist, platform),
|
|
or None when nothing re-pollable resolves."""
|
|
path = Path(source_path)
|
|
sc = find_sidecar(path)
|
|
if sc is None:
|
|
return None
|
|
try:
|
|
data = json.loads(sc.read_text("utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
sd = parse_sidecar(data)
|
|
if not sd.platform:
|
|
return None
|
|
artist_name = derive_top_level_artist(path, import_root)
|
|
if not artist_name:
|
|
return None
|
|
artist = session.execute(
|
|
select(Artist).where(Artist.slug == slugify(artist_name))
|
|
).scalar_one_or_none()
|
|
if artist is None:
|
|
return None
|
|
src = session.execute(
|
|
select(Source)
|
|
.where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == sd.platform,
|
|
Source.enabled.is_(True),
|
|
)
|
|
.order_by(Source.id.asc())
|
|
).scalars().first()
|
|
if src is None:
|
|
return None
|
|
if (src.url or "").startswith("sidecar:"):
|
|
return None # synthetic anchor — not a pollable feed
|
|
return src
|
|
|
|
|
|
def attempt_refetch(
|
|
session: Session, task: ImportTask, import_root: Path,
|
|
) -> dict:
|
|
"""Delete the corrupt file, mark the task refetched, and trigger ONE
|
|
source re-check. Idempotent/bounded: a task already refetched (or
|
|
with no resolvable Source) is a no-op. Commits."""
|
|
if task.refetched:
|
|
return {"status": "already_refetched"}
|
|
src = resolve_refetch_source(session, task.source_path, import_root)
|
|
if src is None:
|
|
return {"status": "no_source"}
|
|
|
|
# Remove the bad copy so gallery-dl's archive-skip re-fetches it on
|
|
# the source re-check instead of skipping the still-present corrupt
|
|
# file.
|
|
try:
|
|
Path(task.source_path).unlink(missing_ok=True)
|
|
except OSError as exc:
|
|
log.warning("refetch unlink failed for %s: %s", task.source_path, exc)
|
|
|
|
task.refetched = True
|
|
session.add(task)
|
|
session.commit()
|
|
|
|
# Lazy import to avoid a tasks→services→tasks import cycle at module
|
|
# load. download_source.delay() is sync-safe in any context.
|
|
from ..tasks.download import download_source
|
|
|
|
download_source.delay(src.id)
|
|
return {"status": "refetch_queued", "source_id": src.id}
|