feat(import-resilience L2): one-shot re-download for corrupt downloaded files
Layer 2 — remediate a corrupt file by re-fetching a fresh copy from its source, bounded to a single attempt. Operator-requested 2026-05-28. New backend/app/services/refetch_service.py: - resolve_refetch_source: parse the failed file's sidecar → platform, derive the artist from the import path, find an ENABLED Source with a real feed URL for (artist, platform). Returns None for filesystem-only imports, missing sidecars, or `sidecar:<platform>:<slug>` synthetic anchors (not pollable). - attempt_refetch: if not already refetched AND a Source resolves, delete the corrupt file (so gallery-dl's skip_existing re-fetches it), set ImportTask.refetched=True, and trigger ONE download_source re-check. Bounded by `refetched` so source-side corruption can't loop. Wiring: - Manual endpoint POST /api/import/tasks/<id>/refetch (only on 'failed' tasks). Returns refetch_queued / no_source / already_refetched / not_found / not_failed. - Auto path in recover_interrupted_tasks: for each poison-pill row, if env FC_AUTO_REFETCH_CORRUPT=1, attempt_refetch (default OFF — the manual button is the primary path; auto is opt-in since re-fetch deletes a file + re-runs the downloader). - Frontend: a cloud-refresh icon button on failed rows in ImportTaskList → stores.import.refetchTask → toast keyed on the result status. Filesystem imports with no upstream return no_source — the operator's only remediation there is replacing the file on disk, surfaced clearly in the toast. Tests: 404 unknown task, 400 non-failed task, no_source when unresolvable, and the full resolvable-source path (file deleted, refetched flag set, one download_source dispatched, second call is a no-op). The resolvable test repoints the migration-seeded import_settings(id=1) scan path rather than inserting a conflicting row.
This commit is contained in:
@@ -171,6 +171,115 @@ async def test_trigger_still_rejects_unknown_mode(client):
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_404_for_unknown_task(client):
|
||||
resp = await client.post("/api/import/tasks/999999/refetch")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_400_for_non_failed_task(client, db):
|
||||
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||
db.add(batch)
|
||||
await db.flush()
|
||||
task = ImportTask(
|
||||
batch_id=batch.id, source_path="/x.jpg", task_type="media",
|
||||
status="complete", finished_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||
assert resp.status_code == 400
|
||||
assert (await resp.get_json())["status"] == "not_failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_no_source_when_unresolvable(client, db):
|
||||
"""A failed task whose file has no sidecar / no resolvable Source
|
||||
returns no_source (filesystem-only import — nothing to re-poll)."""
|
||||
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||
db.add(batch)
|
||||
await db.flush()
|
||||
task = ImportTask(
|
||||
batch_id=batch.id, source_path="/import/nowhere/x.jpg",
|
||||
task_type="media", status="failed", finished_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||
assert resp.status_code == 200
|
||||
assert (await resp.get_json())["status"] == "no_source"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_queued_with_resolvable_source(client, db, tmp_path, monkeypatch):
|
||||
"""A failed task whose file resolves (via sidecar → artist+platform)
|
||||
to an enabled, real-URL Source: the file is deleted, the task is
|
||||
marked refetched, and ONE source re-check is queued."""
|
||||
import json as _json
|
||||
|
||||
from sqlalchemy import update as _update
|
||||
|
||||
from backend.app.models import Artist, ImportSettings, Source
|
||||
from backend.app.tasks import download as download_mod
|
||||
|
||||
# Stub the downloader so the eager test doesn't run a real fetch.
|
||||
dispatched = []
|
||||
monkeypatch.setattr(download_mod.download_source, "delay", dispatched.append)
|
||||
|
||||
# import_root/<ArtistName>/post.jpg + sidecar identifying the platform.
|
||||
import_root = tmp_path / "import"
|
||||
artist_dir = import_root / "Maewix"
|
||||
artist_dir.mkdir(parents=True)
|
||||
media = artist_dir / "post.jpg"
|
||||
media.write_bytes(b"corrupt-bytes")
|
||||
(artist_dir / "post.jpg.json").write_text(
|
||||
_json.dumps({"category": "patreon", "post_id": 123})
|
||||
)
|
||||
|
||||
# import_settings(id=1) is migration-seeded; point its scan path at
|
||||
# our tmp import root rather than inserting a conflicting row.
|
||||
await db.execute(
|
||||
_update(ImportSettings).where(ImportSettings.id == 1)
|
||||
.values(import_scan_path=str(import_root))
|
||||
)
|
||||
artist = Artist(name="Maewix", slug="maewix")
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
db.add(Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://www.patreon.com/maewix", enabled=True,
|
||||
config_overrides={},
|
||||
))
|
||||
batch = ImportBatch(triggered_by="manual", source_path=str(import_root), scan_mode="quick")
|
||||
db.add(batch)
|
||||
await db.flush()
|
||||
task = ImportTask(
|
||||
batch_id=batch.id, source_path=str(media), task_type="media",
|
||||
status="failed", finished_at=datetime.now(UTC),
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["status"] == "refetch_queued"
|
||||
assert len(dispatched) == 1
|
||||
assert not media.exists() # corrupt copy removed for re-fetch
|
||||
|
||||
from sqlalchemy import select as _select
|
||||
refetched = (await db.execute(
|
||||
_select(ImportTask.refetched).where(ImportTask.id == task.id)
|
||||
)).scalar_one()
|
||||
assert refetched is True
|
||||
|
||||
# Second attempt is a no-op (bounded to one).
|
||||
resp2 = await client.post(f"/api/import/tasks/{task.id}/refetch")
|
||||
assert (await resp2.get_json())["status"] == "already_refetched"
|
||||
assert len(dispatched) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_accepts_verify(client, monkeypatch):
|
||||
# Stub the verify task's dispatch so the API contract is asserted
|
||||
|
||||
Reference in New Issue
Block a user