From 513b191e4780f4634c75117466fd5dba7bf3adda Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 19 Apr 2026 14:18:25 -0400 Subject: [PATCH] feat(downloads): capture structured run stats and filtered logs at save time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend foundation for the upcoming Downloads modal refresh. Adds three helpers on GalleryDLService: _compute_run_stats exit_code + downloaded/skipped/per-item-failure/ warning counts, derived from stdout+stderr _extract_errors_warnings filters stderr to just [error]/[warning] lines so the modal can show a concise errors-only view _truncate_log caps a log string (default 500KB) with head+tail+ elision marker, preventing verbose multi-hour runs from bloating the download_sessions JSONB Wires all three into download.metadata_ in the download task. No schema change — everything nests into the existing metadata JSONB column. Follow-up PR will consume these fields in the Downloads.vue modal. --- backend/app/services/gallery_dl.py | 83 +++++++++++++++++++ backend/app/tasks/downloads.py | 17 +++- .../test_gallery_dl_log_processing.py | 57 +++++++++++++ .../services/test_gallery_dl_run_stats.py | 81 ++++++++++++++++++ 4 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 backend/tests/services/test_gallery_dl_log_processing.py create mode 100644 backend/tests/services/test_gallery_dl_run_stats.py diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index cf78560..b83a18f 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -563,6 +563,89 @@ class GalleryDLService: return sum(1 for line in stdout.splitlines() if line.strip().startswith("/")) + def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict: + """Summarize a gallery-dl run as structured counts for UI/analytics. + + Returns a flat dict: + - exit_code: gallery-dl's process return code + - downloaded_count: files actually written (stdout lines starting with '/') + - skipped_count: archive-hit skips (stdout '#' lines + stderr 'skipping ' lines) + - per_item_failures: [download][error] lines (single-item failures the + downloader logs but recovers from) + - warning_count: [*][warning] lines + """ + stdout = stdout or "" + stderr = stderr or "" + + skipped_stdout = sum( + 1 for line in stdout.splitlines() if line.strip().startswith("#") + ) + skipped_stderr = sum( + 1 for line in stderr.splitlines() if "] skipping " in line.lower() + ) + + per_item_failures = sum( + 1 for line in stderr.splitlines() + if "[download][error]" in line.lower() and "failed to download" in line.lower() + ) + + warning_count = sum( + 1 for line in stderr.splitlines() if "][warning]" in line.lower() + ) + + return { + "exit_code": return_code, + "downloaded_count": self._count_downloaded_files(stdout), + "skipped_count": skipped_stdout + skipped_stderr, + "per_item_failures": per_item_failures, + "warning_count": warning_count, + } + + @staticmethod + def _extract_errors_warnings(stderr: str) -> str: + """Filter stderr down to just [error] / [warning] lines. + + Gallery-dl's verbose mode sends all logging to stderr including [debug] + and urllib3 connection-pool noise. This helper produces the short + summary a user actually wants when opening a failed download. + """ + if not stderr: + return "" + + kept = [ + line for line in stderr.splitlines() + if "][error]" in line.lower() or "][warning]" in line.lower() + ] + return "\n".join(kept) + + @staticmethod + def _truncate_log(text: str, max_bytes: int = 500_000) -> str: + """Cap a log field at max_bytes by keeping the head and tail. + + Verbose 10+ minute runs can produce multi-MB stderr blobs. For display + and storage, we keep the first and last halves and drop the middle with + a marker indicating how many lines were elided. Beginning is useful for + context (config, URL); end is useful for the actual failure. + """ + if not text: + return text + + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + + half = max_bytes // 2 + head = encoded[:half].decode("utf-8", errors="ignore") + tail = encoded[-half:].decode("utf-8", errors="ignore") + + head_lines = head.count("\n") + tail_lines = tail.count("\n") + total_lines = text.count("\n") + elided = max(0, total_lines - head_lines - tail_lines) + + marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" + return head + marker + tail + async def download( self, url: str, diff --git a/backend/app/tasks/downloads.py b/backend/app/tasks/downloads.py index c3f0b2e..bab25e1 100644 --- a/backend/app/tasks/downloads.py +++ b/backend/app/tasks/downloads.py @@ -400,10 +400,21 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic # Update download record with results download.completed_at = utcnow() download.file_count = dl_result.files_downloaded - # Store full logs - cleanup task will age out old records + # Persist logs in three forms for the Downloads modal: + # stdout/stderr: raw streams, capped via _truncate_log so huge + # verbose runs don't balloon the JSONB row. + # stderr_errors_warnings: pre-filtered [error]/[warning] lines so + # the UI can show a concise errors-only panel by default. + # run_stats: structured counts for the modal's header row. + run_stats = gdl_service._compute_run_stats( + dl_result.return_code, dl_result.stdout, dl_result.stderr + ) + stderr_summary = gdl_service._extract_errors_warnings(dl_result.stderr) download.metadata_ = { - "stdout": dl_result.stdout if dl_result.stdout else None, - "stderr": dl_result.stderr if dl_result.stderr else None, + "stdout": gdl_service._truncate_log(dl_result.stdout) or None, + "stderr": gdl_service._truncate_log(dl_result.stderr) or None, + "stderr_errors_warnings": stderr_summary or None, + "run_stats": run_stats, "duration_seconds": dl_result.duration_seconds, } diff --git a/backend/tests/services/test_gallery_dl_log_processing.py b/backend/tests/services/test_gallery_dl_log_processing.py new file mode 100644 index 0000000..57538f0 --- /dev/null +++ b/backend/tests/services/test_gallery_dl_log_processing.py @@ -0,0 +1,57 @@ +"""Unit tests for GalleryDLService._extract_errors_warnings and _truncate_log. + +These are the save-time log transforms that feed the Downloads modal's +errors-only panel and prevent multi-MB stderr blobs from landing in JSONB. +""" +from app.services.gallery_dl import GalleryDLService + + +def test_extract_errors_warnings_filters_debug_and_info(): + stderr = ( + "[gallery-dl][debug] Version 1.31.10\n" + "[patreon][info] campaign_id: 548223\n" + "[urllib3.connectionpool][debug] https://example.com 'HEAD /' 200 0\n" + "[download][error] Failed to download 01.part\n" + "[downloader.http][warning] '404 OK' for 'https://example.com'\n" + "[patreon][debug] skipping https://example.com/x\n" + ) + result = GalleryDLService._extract_errors_warnings(stderr) + lines = result.splitlines() + assert len(lines) == 2 + assert "[download][error]" in lines[0] + assert "[downloader.http][warning]" in lines[1] + + +def test_extract_errors_warnings_empty_when_clean_run(): + stderr = ( + "[gallery-dl][debug] Version 1.31.10\n" + "[patreon][debug] skipping https://example.com/x\n" + ) + assert GalleryDLService._extract_errors_warnings(stderr) == "" + + +def test_extract_errors_warnings_handles_empty_input(): + assert GalleryDLService._extract_errors_warnings("") == "" + assert GalleryDLService._extract_errors_warnings(None) == "" + + +def test_truncate_log_passthrough_when_under_limit(): + small = "line\n" * 100 + assert GalleryDLService._truncate_log(small, max_bytes=10_000) == small + + +def test_truncate_log_elides_middle_when_oversized(): + large = ("x" * 100 + "\n") * 2000 # ~200KB + truncated = GalleryDLService._truncate_log(large, max_bytes=10_000) + + assert len(truncated.encode("utf-8")) < len(large.encode("utf-8")) + assert "lines elided" in truncated + assert "bytes" in truncated + # Head and tail preserved + assert truncated.startswith("x") + assert truncated.rstrip().endswith("x") + + +def test_truncate_log_handles_empty_and_none(): + assert GalleryDLService._truncate_log("") == "" + assert GalleryDLService._truncate_log(None) is None diff --git a/backend/tests/services/test_gallery_dl_run_stats.py b/backend/tests/services/test_gallery_dl_run_stats.py new file mode 100644 index 0000000..d81bbfd --- /dev/null +++ b/backend/tests/services/test_gallery_dl_run_stats.py @@ -0,0 +1,81 @@ +"""Unit tests for GalleryDLService._compute_run_stats. + +Covers the structured counts the Downloads modal surfaces in its header row: +exit code, downloaded count, skipped count, per-item failures, warning count. +""" +from unittest.mock import patch + +from app.services.gallery_dl import GalleryDLService + + +def _make_service(): + with patch.object(GalleryDLService, "_load_base_config", return_value={}): + return GalleryDLService(rate_limit=1.0) + + +def test_run_stats_all_zeros_on_empty_output(): + service = _make_service() + stats = service._compute_run_stats(0, "", "") + assert stats == { + "exit_code": 0, + "downloaded_count": 0, + "skipped_count": 0, + "per_item_failures": 0, + "warning_count": 0, + } + + +def test_run_stats_counts_downloads_from_stdout_paths(): + service = _make_service() + stdout = ( + "/data/downloads/Foo/bar/1.jpg\n" + "/data/downloads/Foo/bar/2.jpg\n" + "[some log line that should not count]\n" + ) + stats = service._compute_run_stats(0, stdout, "") + assert stats["downloaded_count"] == 2 + + +def test_run_stats_skips_from_both_streams(): + """stdout '#' lines AND stderr 'skipping ' lines both count.""" + service = _make_service() + stdout = ( + "# /data/downloads/Foo/already_archived_1.jpg\n" + "# /data/downloads/Foo/already_archived_2.jpg\n" + ) + stderr = ( + "[patreon][debug] skipping https://example.com/a.jpg (abc image_large)\n" + "[patreon][debug] skipping https://example.com/b.jpg (def image_large)\n" + "[patreon][debug] skipping https://example.com/c.jpg (ghi image_large)\n" + ) + stats = service._compute_run_stats(0, stdout, stderr) + assert stats["skipped_count"] == 5 + + +def test_run_stats_per_item_failures_counted(): + service = _make_service() + stderr = ( + "[download][error] Failed to download 02_46004845.part\n" + "[download][error] Failed to download 03_46004845.part\n" + "[patreon][debug] something unrelated\n" + ) + stats = service._compute_run_stats(1, "", stderr) + assert stats["per_item_failures"] == 2 + + +def test_run_stats_warnings_counted(): + service = _make_service() + stderr = ( + "[downloader.http][warning] '404 OK' for 'https://example.com/x'\n" + "[patreon][warning] something else\n" + "[patreon][error] not a warning\n" + ) + stats = service._compute_run_stats(1, "", stderr) + assert stats["warning_count"] == 2 + + +def test_run_stats_exit_code_passed_through(): + service = _make_service() + assert service._compute_run_stats(0, "", "")["exit_code"] == 0 + assert service._compute_run_stats(4, "", "")["exit_code"] == 4 + assert service._compute_run_stats(-9, "", "")["exit_code"] == -9