feat(downloader): validate and quarantine truncated files post-download
After gallery-dl returns, parse stdout for written file paths and run
the magic-byte validator on each. Files that fail are moved to
{download_path}/_quarantine/{subscription}/{platform}/ with a sidecar
JSON capturing the source URL, validation reason, and original path —
enough for an operator to redownload-from-source or delete.
Adds ErrorType.VALIDATION_FAILED. A successful gallery-dl run that
produced quarantined files is now a soft failure (success=False,
error_message names the dominant failure reason and count) so the
source's error_count ticks up and the dashboard surfaces it. Gated
by download.validate_files setting (default True).
files_quarantined and quarantined_paths are persisted into download
metadata for the UI/API to consume.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -319,6 +319,9 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
|
||||
# Get rate limit from database settings
|
||||
db_rate_limit = await _get_db_setting(session, "download.rate_limit", settings.download_rate_limit)
|
||||
db_validate_files = bool(
|
||||
await _get_db_setting(session, "download.validate_files", True)
|
||||
)
|
||||
|
||||
# Session is now closed - DB connection released
|
||||
finally:
|
||||
@@ -331,7 +334,10 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
resolved_campaign_id: Optional[str] = None
|
||||
|
||||
try:
|
||||
gdl_service = GalleryDLService(rate_limit=db_rate_limit)
|
||||
gdl_service = GalleryDLService(
|
||||
rate_limit=db_rate_limit,
|
||||
validate_files=db_validate_files,
|
||||
)
|
||||
attempt = await _execute_download_with_retry(
|
||||
gdl_service=gdl_service,
|
||||
platform=source_platform,
|
||||
@@ -409,6 +415,7 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
run_stats = gdl_service._compute_run_stats(
|
||||
dl_result.return_code, dl_result.stdout, dl_result.stderr
|
||||
)
|
||||
run_stats["quarantined_count"] = dl_result.files_quarantined
|
||||
stderr_summary = gdl_service._extract_errors_warnings(dl_result.stderr)
|
||||
download.metadata_ = {
|
||||
"stdout": gdl_service._truncate_log(dl_result.stdout) or None,
|
||||
@@ -416,6 +423,7 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
"stderr_errors_warnings": stderr_summary or None,
|
||||
"run_stats": run_stats,
|
||||
"duration_seconds": dl_result.duration_seconds,
|
||||
"quarantined_paths": dl_result.quarantined_paths or None,
|
||||
}
|
||||
|
||||
if dl_result.success:
|
||||
|
||||
Reference in New Issue
Block a user