bvandeusen
dcfe55d731
Layer 2 — remediate a corrupt file by re-fetching a fresh copy from its source, bounded to a single attempt. Operator-requested 2026-05-28. New backend/app/services/refetch_service.py: - resolve_refetch_source: parse the failed file's sidecar → platform, derive the artist from the import path, find an ENABLED Source with a real feed URL for (artist, platform). Returns None for filesystem-only imports, missing sidecars, or `sidecar:<platform>:<slug>` synthetic anchors (not pollable). - attempt_refetch: if not already refetched AND a Source resolves, delete the corrupt file (so gallery-dl's skip_existing re-fetches it), set ImportTask.refetched=True, and trigger ONE download_source re-check. Bounded by `refetched` so source-side corruption can't loop. Wiring: - Manual endpoint POST /api/import/tasks/<id>/refetch (only on 'failed' tasks). Returns refetch_queued / no_source / already_refetched / not_found / not_failed. - Auto path in recover_interrupted_tasks: for each poison-pill row, if env FC_AUTO_REFETCH_CORRUPT=1, attempt_refetch (default OFF — the manual button is the primary path; auto is opt-in since re-fetch deletes a file + re-runs the downloader). - Frontend: a cloud-refresh icon button on failed rows in ImportTaskList → stores.import.refetchTask → toast keyed on the result status. Filesystem imports with no upstream return no_source — the operator's only remediation there is replacing the file on disk, surfaced clearly in the toast. Tests: 404 unknown task, 400 non-failed task, no_source when unresolvable, and the full resolvable-source path (file deleted, refetched flag set, one download_source dispatched, second call is a no-op). The resolvable test repoints the migration-seeded import_settings(id=1) scan path rather than inserting a conflicting row.
107 lines
3.5 KiB
Python
107 lines
3.5 KiB
Python
"""Layer-2 one-shot re-download remediation for corrupt imported files.
|
|
|
|
When an import fails on a file that came from a known, pollable
|
|
subscription Source, deleting the bad copy and re-running the source's
|
|
downloader can fetch a fresh, unblemished copy. This only helps when:
|
|
|
|
- the corruption is in transit / on disk (not at the source), AND
|
|
- the file resolves to an ENABLED Source with a real feed URL
|
|
(a `sidecar:<platform>:<slug>` synthetic anchor is not pollable),
|
|
AND
|
|
- we haven't already re-fetched this task once (bounded by
|
|
ImportTask.refetched so source-side corruption can't loop).
|
|
|
|
Filesystem-only imports with no resolvable Source return 'no_source' —
|
|
the operator's only remediation there is to replace the file on disk.
|
|
|
|
Operator-requested 2026-05-28 (Layer 2).
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..models import Artist, ImportTask, Source
|
|
from ..utils.paths import derive_top_level_artist
|
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
|
from ..utils.slug import slugify
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def resolve_refetch_source(
|
|
session: Session, source_path: str, import_root: Path,
|
|
) -> Source | None:
|
|
"""Find an enabled, real-URL Source for the file's (artist, platform),
|
|
or None when nothing re-pollable resolves."""
|
|
path = Path(source_path)
|
|
sc = find_sidecar(path)
|
|
if sc is None:
|
|
return None
|
|
try:
|
|
data = json.loads(sc.read_text("utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
sd = parse_sidecar(data)
|
|
if not sd.platform:
|
|
return None
|
|
artist_name = derive_top_level_artist(path, import_root)
|
|
if not artist_name:
|
|
return None
|
|
artist = session.execute(
|
|
select(Artist).where(Artist.slug == slugify(artist_name))
|
|
).scalar_one_or_none()
|
|
if artist is None:
|
|
return None
|
|
src = session.execute(
|
|
select(Source)
|
|
.where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == sd.platform,
|
|
Source.enabled.is_(True),
|
|
)
|
|
.order_by(Source.id.asc())
|
|
).scalars().first()
|
|
if src is None:
|
|
return None
|
|
if (src.url or "").startswith("sidecar:"):
|
|
return None # synthetic anchor — not a pollable feed
|
|
return src
|
|
|
|
|
|
def attempt_refetch(
|
|
session: Session, task: ImportTask, import_root: Path,
|
|
) -> dict:
|
|
"""Delete the corrupt file, mark the task refetched, and trigger ONE
|
|
source re-check. Idempotent/bounded: a task already refetched (or
|
|
with no resolvable Source) is a no-op. Commits."""
|
|
if task.refetched:
|
|
return {"status": "already_refetched"}
|
|
src = resolve_refetch_source(session, task.source_path, import_root)
|
|
if src is None:
|
|
return {"status": "no_source"}
|
|
|
|
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
|
# the source re-check instead of skipping the still-present corrupt
|
|
# file.
|
|
try:
|
|
Path(task.source_path).unlink(missing_ok=True)
|
|
except OSError as exc:
|
|
log.warning("refetch unlink failed for %s: %s", task.source_path, exc)
|
|
|
|
task.refetched = True
|
|
session.add(task)
|
|
session.commit()
|
|
|
|
# Lazy import to avoid a tasks→services→tasks import cycle at module
|
|
# load. download_source.delay() is sync-safe in any context.
|
|
from ..tasks.download import download_source
|
|
|
|
download_source.delay(src.id)
|
|
return {"status": "refetch_queued", "source_id": src.id}
|