feat(downloads): capture structured run stats and filtered logs at save time
Backend foundation for the upcoming Downloads modal refresh. Adds three
helpers on GalleryDLService:
_compute_run_stats exit_code + downloaded/skipped/per-item-failure/
warning counts, derived from stdout+stderr
_extract_errors_warnings filters stderr to just [error]/[warning] lines
so the modal can show a concise errors-only view
_truncate_log caps a log string (default 500KB) with head+tail+
elision marker, preventing verbose multi-hour runs
from bloating the download_sessions JSONB
Wires all three into download.metadata_ in the download task. No schema
change — everything nests into the existing metadata JSONB column.
Follow-up PR will consume these fields in the Downloads.vue modal.
This commit is contained in:
@@ -563,6 +563,89 @@ class GalleryDLService:
|
||||
|
||||
return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))
|
||||
|
||||
def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
|
||||
"""Summarize a gallery-dl run as structured counts for UI/analytics.
|
||||
|
||||
Returns a flat dict:
|
||||
- exit_code: gallery-dl's process return code
|
||||
- downloaded_count: files actually written (stdout lines starting with '/')
|
||||
- skipped_count: archive-hit skips (stdout '#' lines + stderr 'skipping ' lines)
|
||||
- per_item_failures: [download][error] lines (single-item failures the
|
||||
downloader logs but recovers from)
|
||||
- warning_count: [*][warning] lines
|
||||
"""
|
||||
stdout = stdout or ""
|
||||
stderr = stderr or ""
|
||||
|
||||
skipped_stdout = sum(
|
||||
1 for line in stdout.splitlines() if line.strip().startswith("#")
|
||||
)
|
||||
skipped_stderr = sum(
|
||||
1 for line in stderr.splitlines() if "] skipping " in line.lower()
|
||||
)
|
||||
|
||||
per_item_failures = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "[download][error]" in line.lower() and "failed to download" in line.lower()
|
||||
)
|
||||
|
||||
warning_count = sum(
|
||||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||||
)
|
||||
|
||||
return {
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": self._count_downloaded_files(stdout),
|
||||
"skipped_count": skipped_stdout + skipped_stderr,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_errors_warnings(stderr: str) -> str:
|
||||
"""Filter stderr down to just [error] / [warning] lines.
|
||||
|
||||
Gallery-dl's verbose mode sends all logging to stderr including [debug]
|
||||
and urllib3 connection-pool noise. This helper produces the short
|
||||
summary a user actually wants when opening a failed download.
|
||||
"""
|
||||
if not stderr:
|
||||
return ""
|
||||
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
"""Cap a log field at max_bytes by keeping the head and tail.
|
||||
|
||||
Verbose 10+ minute runs can produce multi-MB stderr blobs. For display
|
||||
and storage, we keep the first and last halves and drop the middle with
|
||||
a marker indicating how many lines were elided. Beginning is useful for
|
||||
context (config, URL); end is useful for the actual failure.
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
async def download(
|
||||
self,
|
||||
url: str,
|
||||
|
||||
@@ -400,10 +400,21 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
# Update download record with results
|
||||
download.completed_at = utcnow()
|
||||
download.file_count = dl_result.files_downloaded
|
||||
# Store full logs - cleanup task will age out old records
|
||||
# Persist logs in three forms for the Downloads modal:
|
||||
# stdout/stderr: raw streams, capped via _truncate_log so huge
|
||||
# verbose runs don't balloon the JSONB row.
|
||||
# stderr_errors_warnings: pre-filtered [error]/[warning] lines so
|
||||
# the UI can show a concise errors-only panel by default.
|
||||
# run_stats: structured counts for the modal's header row.
|
||||
run_stats = gdl_service._compute_run_stats(
|
||||
dl_result.return_code, dl_result.stdout, dl_result.stderr
|
||||
)
|
||||
stderr_summary = gdl_service._extract_errors_warnings(dl_result.stderr)
|
||||
download.metadata_ = {
|
||||
"stdout": dl_result.stdout if dl_result.stdout else None,
|
||||
"stderr": dl_result.stderr if dl_result.stderr else None,
|
||||
"stdout": gdl_service._truncate_log(dl_result.stdout) or None,
|
||||
"stderr": gdl_service._truncate_log(dl_result.stderr) or None,
|
||||
"stderr_errors_warnings": stderr_summary or None,
|
||||
"run_stats": run_stats,
|
||||
"duration_seconds": dl_result.duration_seconds,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user