feat(downloads): capture structured run stats and filtered logs at save time

Backend foundation for the upcoming Downloads modal refresh. Adds three
helpers on GalleryDLService:

  _compute_run_stats   exit_code + downloaded/skipped/per-item-failure/
                      warning counts, derived from stdout+stderr
  _extract_errors_warnings   filters stderr to just [error]/[warning] lines
                             so the modal can show a concise errors-only view
  _truncate_log        caps a log string (default 500KB) with head+tail+
                      elision marker, preventing verbose multi-hour runs
                      from bloating the download_sessions JSONB

Wires all three into download.metadata_ in the download task. No schema
change — everything nests into the existing metadata JSONB column.

Follow-up PR will consume these fields in the Downloads.vue modal.
This commit is contained in:
2026-04-19 14:18:25 -04:00
parent edbb349c58
commit 513b191e47
4 changed files with 235 additions and 3 deletions
+83
View File
@@ -563,6 +563,89 @@ class GalleryDLService:
return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))
def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
"""Summarize a gallery-dl run as structured counts for UI/analytics.
Returns a flat dict:
- exit_code: gallery-dl's process return code
- downloaded_count: files actually written (stdout lines starting with '/')
- skipped_count: archive-hit skips (stdout '#' lines + stderr 'skipping ' lines)
- per_item_failures: [download][error] lines (single-item failures the
downloader logs but recovers from)
- warning_count: [*][warning] lines
"""
stdout = stdout or ""
stderr = stderr or ""
skipped_stdout = sum(
1 for line in stdout.splitlines() if line.strip().startswith("#")
)
skipped_stderr = sum(
1 for line in stderr.splitlines() if "] skipping " in line.lower()
)
per_item_failures = sum(
1 for line in stderr.splitlines()
if "[download][error]" in line.lower() and "failed to download" in line.lower()
)
warning_count = sum(
1 for line in stderr.splitlines() if "][warning]" in line.lower()
)
return {
"exit_code": return_code,
"downloaded_count": self._count_downloaded_files(stdout),
"skipped_count": skipped_stdout + skipped_stderr,
"per_item_failures": per_item_failures,
"warning_count": warning_count,
}
@staticmethod
def _extract_errors_warnings(stderr: str) -> str:
"""Filter stderr down to just [error] / [warning] lines.
Gallery-dl's verbose mode sends all logging to stderr including [debug]
and urllib3 connection-pool noise. This helper produces the short
summary a user actually wants when opening a failed download.
"""
if not stderr:
return ""
kept = [
line for line in stderr.splitlines()
if "][error]" in line.lower() or "][warning]" in line.lower()
]
return "\n".join(kept)
@staticmethod
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
"""Cap a log field at max_bytes by keeping the head and tail.
Verbose 10+ minute runs can produce multi-MB stderr blobs. For display
and storage, we keep the first and last halves and drop the middle with
a marker indicating how many lines were elided. Beginning is useful for
context (config, URL); end is useful for the actual failure.
"""
if not text:
return text
encoded = text.encode("utf-8")
if len(encoded) <= max_bytes:
return text
half = max_bytes // 2
head = encoded[:half].decode("utf-8", errors="ignore")
tail = encoded[-half:].decode("utf-8", errors="ignore")
head_lines = head.count("\n")
tail_lines = tail.count("\n")
total_lines = text.count("\n")
elided = max(0, total_lines - head_lines - tail_lines)
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
return head + marker + tail
async def download(
self,
url: str,
+14 -3
View File
@@ -400,10 +400,21 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
# Update download record with results
download.completed_at = utcnow()
download.file_count = dl_result.files_downloaded
# Store full logs - cleanup task will age out old records
# Persist logs in three forms for the Downloads modal:
# stdout/stderr: raw streams, capped via _truncate_log so huge
# verbose runs don't balloon the JSONB row.
# stderr_errors_warnings: pre-filtered [error]/[warning] lines so
# the UI can show a concise errors-only panel by default.
# run_stats: structured counts for the modal's header row.
run_stats = gdl_service._compute_run_stats(
dl_result.return_code, dl_result.stdout, dl_result.stderr
)
stderr_summary = gdl_service._extract_errors_warnings(dl_result.stderr)
download.metadata_ = {
"stdout": dl_result.stdout if dl_result.stdout else None,
"stderr": dl_result.stderr if dl_result.stderr else None,
"stdout": gdl_service._truncate_log(dl_result.stdout) or None,
"stderr": gdl_service._truncate_log(dl_result.stderr) or None,
"stderr_errors_warnings": stderr_summary or None,
"run_stats": run_stats,
"duration_seconds": dl_result.duration_seconds,
}