feat(downloads): capture structured run stats and filtered logs at save time

Backend foundation for the upcoming Downloads modal refresh. Adds three
helpers on GalleryDLService:

  _compute_run_stats   exit_code + downloaded/skipped/per-item-failure/
                      warning counts, derived from stdout+stderr
  _extract_errors_warnings   filters stderr to just [error]/[warning] lines
                             so the modal can show a concise errors-only view
  _truncate_log        caps a log string (default 500KB) with head+tail+
                      elision marker, preventing verbose multi-hour runs
                      from bloating the download_sessions JSONB

Wires all three into download.metadata_ in the download task. No schema
change — everything nests into the existing metadata JSONB column.

Follow-up PR will consume these fields in the Downloads.vue modal.
This commit is contained in:
2026-04-19 14:18:25 -04:00
parent edbb349c58
commit 513b191e47
4 changed files with 235 additions and 3 deletions
@@ -0,0 +1,57 @@
"""Unit tests for GalleryDLService._extract_errors_warnings and _truncate_log.
These are the save-time log transforms that feed the Downloads modal's
errors-only panel and prevent multi-MB stderr blobs from landing in JSONB.
"""
from app.services.gallery_dl import GalleryDLService
def test_extract_errors_warnings_filters_debug_and_info():
stderr = (
"[gallery-dl][debug] Version 1.31.10\n"
"[patreon][info] campaign_id: 548223\n"
"[urllib3.connectionpool][debug] https://example.com 'HEAD /' 200 0\n"
"[download][error] Failed to download 01.part\n"
"[downloader.http][warning] '404 OK' for 'https://example.com'\n"
"[patreon][debug] skipping https://example.com/x\n"
)
result = GalleryDLService._extract_errors_warnings(stderr)
lines = result.splitlines()
assert len(lines) == 2
assert "[download][error]" in lines[0]
assert "[downloader.http][warning]" in lines[1]
def test_extract_errors_warnings_empty_when_clean_run():
stderr = (
"[gallery-dl][debug] Version 1.31.10\n"
"[patreon][debug] skipping https://example.com/x\n"
)
assert GalleryDLService._extract_errors_warnings(stderr) == ""
def test_extract_errors_warnings_handles_empty_input():
assert GalleryDLService._extract_errors_warnings("") == ""
assert GalleryDLService._extract_errors_warnings(None) == ""
def test_truncate_log_passthrough_when_under_limit():
small = "line\n" * 100
assert GalleryDLService._truncate_log(small, max_bytes=10_000) == small
def test_truncate_log_elides_middle_when_oversized():
large = ("x" * 100 + "\n") * 2000 # ~200KB
truncated = GalleryDLService._truncate_log(large, max_bytes=10_000)
assert len(truncated.encode("utf-8")) < len(large.encode("utf-8"))
assert "lines elided" in truncated
assert "bytes" in truncated
# Head and tail preserved
assert truncated.startswith("x")
assert truncated.rstrip().endswith("x")
def test_truncate_log_handles_empty_and_none():
assert GalleryDLService._truncate_log("") == ""
assert GalleryDLService._truncate_log(None) is None
@@ -0,0 +1,81 @@
"""Unit tests for GalleryDLService._compute_run_stats.
Covers the structured counts the Downloads modal surfaces in its header row:
exit code, downloaded count, skipped count, per-item failures, warning count.
"""
from unittest.mock import patch
from app.services.gallery_dl import GalleryDLService
def _make_service():
with patch.object(GalleryDLService, "_load_base_config", return_value={}):
return GalleryDLService(rate_limit=1.0)
def test_run_stats_all_zeros_on_empty_output():
service = _make_service()
stats = service._compute_run_stats(0, "", "")
assert stats == {
"exit_code": 0,
"downloaded_count": 0,
"skipped_count": 0,
"per_item_failures": 0,
"warning_count": 0,
}
def test_run_stats_counts_downloads_from_stdout_paths():
service = _make_service()
stdout = (
"/data/downloads/Foo/bar/1.jpg\n"
"/data/downloads/Foo/bar/2.jpg\n"
"[some log line that should not count]\n"
)
stats = service._compute_run_stats(0, stdout, "")
assert stats["downloaded_count"] == 2
def test_run_stats_skips_from_both_streams():
"""stdout '#' lines AND stderr 'skipping ' lines both count."""
service = _make_service()
stdout = (
"# /data/downloads/Foo/already_archived_1.jpg\n"
"# /data/downloads/Foo/already_archived_2.jpg\n"
)
stderr = (
"[patreon][debug] skipping https://example.com/a.jpg (abc image_large)\n"
"[patreon][debug] skipping https://example.com/b.jpg (def image_large)\n"
"[patreon][debug] skipping https://example.com/c.jpg (ghi image_large)\n"
)
stats = service._compute_run_stats(0, stdout, stderr)
assert stats["skipped_count"] == 5
def test_run_stats_per_item_failures_counted():
service = _make_service()
stderr = (
"[download][error] Failed to download 02_46004845.part\n"
"[download][error] Failed to download 03_46004845.part\n"
"[patreon][debug] something unrelated\n"
)
stats = service._compute_run_stats(1, "", stderr)
assert stats["per_item_failures"] == 2
def test_run_stats_warnings_counted():
service = _make_service()
stderr = (
"[downloader.http][warning] '404 OK' for 'https://example.com/x'\n"
"[patreon][warning] something else\n"
"[patreon][error] not a warning\n"
)
stats = service._compute_run_stats(1, "", stderr)
assert stats["warning_count"] == 2
def test_run_stats_exit_code_passed_through():
service = _make_service()
assert service._compute_run_stats(0, "", "")["exit_code"] == 0
assert service._compute_run_stats(4, "", "")["exit_code"] == 4
assert service._compute_run_stats(-9, "", "")["exit_code"] == -9