From 3ee7de7ecd3ab07029355e35f3f1ff9279298240 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 30 Jan 2026 22:04:07 -0500 Subject: [PATCH] tuning error recognition --- .claude/settings.local.json | 5 +- .gitignore | 7 + backend/app/api/downloads.py | 164 +++++++++++++++++- backend/app/services/gallery_dl.py | 106 +++++++++--- backend/scripts/export_failed_logs.py | 232 ++++++++++++++++++++++++++ frontend/src/views/Downloads.vue | 20 +++ summary.md | 43 ++++- 7 files changed, 546 insertions(+), 31 deletions(-) create mode 100644 backend/scripts/export_failed_logs.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 8278838..d41d16e 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -14,7 +14,10 @@ "Bash(where npm:*)", "Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")", "WebFetch(domain:github.com)", - "WebFetch(domain:raw.githubusercontent.com)" + "WebFetch(domain:raw.githubusercontent.com)", + "Bash(python -m py_compile:*)", + "Bash(powershell:*)", + "Bash(python:*)" ] } } diff --git a/.gitignore b/.gitignore index 840d53f..cb41b21 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,10 @@ backend/static/ # Extension packages *.xpi extension-dist/ + +# Analysis and debugging artifacts +analysis/ +**/analysis_*.json +**/analysis_*.txt +**/failed_logs_export_*.json +ERROR_RECOGNITION_ANALYSIS.md diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py index 7b32901..7d8fafe 100644 --- a/backend/app/api/downloads.py +++ b/backend/app/api/downloads.py @@ -1,9 +1,11 @@ """Download history API endpoints.""" -from quart import Blueprint, request, jsonify, current_app -from sqlalchemy import select, func, and_ +from datetime import datetime, timedelta +from quart import Blueprint, request, jsonify, current_app, Response +from sqlalchemy import select, func, and_, or_, desc from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload +import json from app.models.base import utcnow from app.models.download import Download, DownloadStatus @@ -285,3 +287,161 @@ async def get_stats(): "queued": status_counts.get(DownloadStatus.QUEUED, 0), "running": status_counts.get(DownloadStatus.RUNNING, 0), }) + + +@bp.route("/export-failed-logs", methods=["GET"]) +async def export_failed_logs(): + """Export failed download logs for error classification analysis. + + Returns a JSON file that can be fed to Claude to improve error recognition patterns. + + Query params: + limit: Maximum number of records (default 50, max 200) + days: Only include failures from last N days (default 30) + error_type: Filter by specific error type (e.g., not_found, auth_error) + include_success: Also include some successful "no new content" for comparison + """ + limit = min(int(request.args.get("limit", 50)), 200) + days = int(request.args.get("days", 30)) + error_type = request.args.get("error_type") + include_success = request.args.get("include_success", "").lower() in ("true", "1", "yes") + + cutoff_date = utcnow() - timedelta(days=days) + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Build query for failed downloads (eager load source for platform) + query = select(Download).options(selectinload(Download.source)).where( + and_( + Download.status == DownloadStatus.FAILED, + Download.created_at >= cutoff_date, + ) + ) + + if error_type: + query = query.where(Download.error_type == error_type) + + # Order by most recent first + query = query.order_by(desc(Download.created_at)).limit(limit) + result = await session.execute(query) + failed_downloads = result.scalars().all() + + # Optionally include some successful NO_NEW_CONTENT for comparison + success_downloads = [] + if include_success: + success_query = select(Download).options(selectinload(Download.source)).where( + and_( + Download.status == DownloadStatus.COMPLETED, + Download.created_at >= cutoff_date, + Download.file_count == 0, # No new content cases + ) + ).order_by(desc(Download.created_at)).limit(limit // 4) + success_result = await session.execute(success_query) + success_downloads = success_result.scalars().all() + + # Format for export + exports = [] + + for download in failed_downloads: + exports.append(_format_download_for_export(download, "failed")) + + for download in success_downloads: + exports.append(_format_download_for_export(download, "success_no_new_content")) + + # Build output document + output = { + "export_info": { + "exported_at": utcnow().isoformat(), + "total_records": len(exports), + "failed_count": len(failed_downloads), + "success_comparison_count": len(success_downloads), + "days_included": days, + "error_type_filter": error_type, + }, + "analysis_prompt": _generate_analysis_prompt(), + "downloads": exports, + } + + # Return as downloadable JSON file + response = Response( + json.dumps(output, indent=2, default=str), + mimetype="application/json", + headers={ + "Content-Disposition": f"attachment; filename=failed_logs_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + } + ) + return response + + +def _format_download_for_export(download: Download, classification: str) -> dict: + """Format a download record for export.""" + # Extract logs from metadata + metadata = download.metadata_ or {} + stdout = metadata.get("stdout") or "" + stderr = metadata.get("stderr") or "" + + # Truncate very long logs but keep enough for analysis + max_log_length = 50000 # 50KB per log field + if stdout and len(stdout) > max_log_length: + stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]" + if stderr and len(stderr) > max_log_length: + stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]" + + # Get platform from source if available + platform = None + if download.source: + platform = download.source.platform + + return { + "id": download.id, + "classification": classification, + "assigned_error_type": download.error_type, + "assigned_error_message": download.error_message, + "platform": platform, + "url": download.url, + "status": download.status, # Already a string, not enum + "file_count": download.file_count, + "created_at": download.created_at.isoformat() if download.created_at else None, + "duration_seconds": metadata.get("duration_seconds"), + "logs": { + "stdout": stdout, + "stderr": stderr, + }, + "user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type" + "suggested_error_type": None, # User can suggest what it should be + "notes": None, # User can add context + } + + +def _generate_analysis_prompt() -> str: + """Generate a prompt for Claude to analyze the logs.""" + return """ +## Error Log Analysis Task + +I'm providing you with download logs from a gallery-dl based downloader. Each record includes: +- The error_type that was assigned by the current classification logic +- The actual stdout/stderr logs from gallery-dl +- Platform and URL information + +Please analyze these logs and identify: + +1. **False Positives**: Cases where an error was reported but the download actually succeeded + - Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked + +2. **False Negatives**: Cases classified as one error type that should be another + - Example: classified as "not_found" but logs show authentication issues + +3. **Pattern Improvements**: Suggest more specific patterns to detect each error type + - Current patterns may be too broad (matching debug output) or too narrow (missing variations) + +4. **New Error Types**: Any failure modes not currently categorized + +For each issue found, provide: +- The download ID +- Current classification vs suggested classification +- The specific log lines that support your analysis +- Suggested pattern improvements (if applicable) + +Focus on actionable improvements to the _categorize_error() function in gallery_dl.py. +""" diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 06a2619..e84ae99 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -316,23 +316,52 @@ class GalleryDLService: return config def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]: - """Categorize the error based on output and return code.""" + """Categorize the error based on output and return code. + + Priority order: + 1. Check for skip messages (# prefixed lines or text patterns) → NO_NEW_CONTENT + 2. Check for actual errors (auth, rate limit, 404, network, etc.) + 3. Fall back to UNKNOWN_ERROR + + This ordering prevents false positives where debug output matches error patterns. + """ combined = f"{stdout} {stderr}".lower() - # Check for "no new content" FIRST for any non-zero return code - # Gallery-dl returns 1 when all files are skipped or no new content found - # This check must run before other error checks to avoid false positives - skip_indicators = ["skipping", "skip", "already exists", "archive"] - if any(indicator in combined for indicator in skip_indicators): - # If we see skip messages and no actual error messages, it's just "no new content" - actual_errors = ["error", "failed", "exception", "traceback"] - if not any(err in combined for err in actual_errors): - return ErrorType.NO_NEW_CONTENT, "No new content to download" + # === PHASE 1: Detect skip/no-new-content scenarios === + # Gallery-dl indicates skipped files two ways: + # 1. Lines starting with '#' in stdout (e.g., "# /data/downloads/Artist/file.png") + # 2. Text patterns like "skipping", "already exists" + + # Count lines starting with '#' (skip messages) + skip_line_count = len([ + line for line in stdout.split('\n') + if line.strip().startswith('#') + ]) + + # Text-based skip indicators + skip_text_indicators = ["skipping", "already exists", "archive"] + has_skip_text = any(ind in combined for ind in skip_text_indicators) + + # Actual error indicators that override skip detection + actual_error_indicators = [ + "][error]", # gallery-dl error log level: [patreon][error] + "exception", # Python exceptions + "traceback", # Python tracebacks + "download failed", + "extraction failed", + ] + has_actual_error = any(err in combined for err in actual_error_indicators) + + # If we have skip evidence and no real errors, it's no_new_content + if (skip_line_count > 0 or has_skip_text) and not has_actual_error: + return ErrorType.NO_NEW_CONTENT, "No new content to download" # Empty output with low return codes often means nothing to download if return_code in (0, 1) and not stdout.strip(): return ErrorType.NO_NEW_CONTENT, "No new content to download" + # === PHASE 2: Check for specific error types === + # Auth errors - look for error context, not just HTTP codes that appear in debug logs # HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors auth_error_patterns = [ @@ -370,29 +399,50 @@ class GalleryDLService: if any(pattern in combined for pattern in rate_limit_patterns): return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time" - # 404 Not Found - check for HTTP 404 responses, not just "404" anywhere + # 404 Not Found - check for HTTP 404 responses with error context + # Be specific to avoid matching "not found" in debug messages like "archive entry not found" not_found_patterns = [ - '" 404 ', # HTTP response - "error 404", - "status: 404", - "not found", - "does not exist", - "no longer available", + '" 404 ', # HTTP response log line + "error 404", # Explicit error 404 + "status: 404", # Status code indicator + "status code: 404", # Status code indicator + "404 not found", # Standard 404 message + "page not found", # Common 404 page text + "user not found", # User doesn't exist + "creator not found", + "artist not found", + "content no longer available", + "has been deleted", + "account.*deleted", + "profile.*not.*exist", ] - if any(pattern in combined for pattern in not_found_patterns): + # Also require error-level context for generic patterns to avoid false positives + generic_not_found = ["does not exist", "no longer available"] + has_specific_404 = any(pattern in combined for pattern in not_found_patterns) + has_generic_with_error = any( + pattern in combined and "][error]" in combined + for pattern in generic_not_found + ) + if has_specific_404 or has_generic_with_error: return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content" - # Network errors + # Network errors - use specific patterns to avoid matching config values + # (e.g., "timeout": 30.0 in debug output should not trigger network_error) network_error_patterns = [ - "timeout", - "timed out", + "timed out", # Actual timeout occurred + "read timed out", # urllib3/requests timeout + "connect timed out", # Connection timeout + "connection timed out", "connection refused", "connection reset", "network unreachable", - "name resolution", - "dns", - "ssl error", - "certificate verify", + "name resolution failed", + "name or service not known", + "nodename nor servname provided", + "ssl: certificate_verify_failed", + "ssl: wrong_version_number", + "certificate verify failed", + "[errno", # Socket errors like [Errno 111] ] if any(pattern in combined for pattern in network_error_patterns): return ErrorType.NETWORK_ERROR, "Network error - check internet connection" @@ -418,10 +468,12 @@ class GalleryDLService: if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" - # If we got here with return code 1 and saw some activity, it might just be no new content - if return_code == 1: + # If we got here with return code 1 and saw some activity but no error patterns matched, + # it's likely just no new content (gallery-dl returns 1 when nothing to download) + if return_code == 1 and stdout.strip() and not has_actual_error: return ErrorType.NO_NEW_CONTENT, "No new content to download" + # Final fallback - genuine unknown error return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" def _count_downloaded_files(self, stdout: str) -> int: diff --git a/backend/scripts/export_failed_logs.py b/backend/scripts/export_failed_logs.py new file mode 100644 index 0000000..651e116 --- /dev/null +++ b/backend/scripts/export_failed_logs.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Export failed download logs for error classification analysis. + +Outputs a JSON file that can be fed to Claude to improve error recognition patterns. + +Usage: + python scripts/export_failed_logs.py [options] + +Options: + --limit N Maximum number of records to export (default: 50) + --days N Only include failures from last N days (default: 30) + --error-type TYPE Filter by specific error type (e.g., not_found, auth_error) + --output FILE Output file path (default: failed_logs_export.json) + --include-success Also include some successful "no new content" for comparison +""" + +import argparse +import json +import sys +from datetime import datetime, timedelta +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from sqlalchemy import create_engine, desc, or_ +from sqlalchemy.orm import sessionmaker + +from app.models.download import Download, DownloadStatus +from app.config import get_settings + + +def export_failed_logs( + limit: int = 50, + days: int = 30, + error_type: str | None = None, + include_success: bool = False, + output_path: str = "failed_logs_export.json", +): + """Export failed download logs to JSON for analysis.""" + + settings = get_settings() + engine = create_engine(settings.database_url) + Session = sessionmaker(bind=engine) + session = Session() + + try: + cutoff_date = datetime.utcnow() - timedelta(days=days) + + # Build query for failed downloads + query = session.query(Download).filter( + Download.status == DownloadStatus.FAILED, + Download.created_at >= cutoff_date, + ) + + if error_type: + query = query.filter(Download.error_type == error_type) + + # Order by most recent first + failed_downloads = query.order_by(desc(Download.created_at)).limit(limit).all() + + # Optionally include some successful NO_NEW_CONTENT for comparison + success_downloads = [] + if include_success: + success_query = session.query(Download).filter( + Download.status == DownloadStatus.COMPLETED, + Download.created_at >= cutoff_date, + Download.file_count == 0, # No new content cases + ).order_by(desc(Download.created_at)).limit(limit // 4) + success_downloads = success_query.all() + + # Format for export + exports = [] + + for download in failed_downloads: + exports.append(format_download_for_export(download, "failed")) + + for download in success_downloads: + exports.append(format_download_for_export(download, "success_no_new_content")) + + # Build output document + output = { + "export_info": { + "exported_at": datetime.utcnow().isoformat(), + "total_records": len(exports), + "failed_count": len(failed_downloads), + "success_comparison_count": len(success_downloads), + "days_included": days, + "error_type_filter": error_type, + }, + "analysis_prompt": generate_analysis_prompt(), + "downloads": exports, + } + + # Write to file + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2, default=str) + + print(f"Exported {len(exports)} records to {output_path}") + print(f" - Failed: {len(failed_downloads)}") + if include_success: + print(f" - Success (no new content): {len(success_downloads)}") + + # Print summary by error type + error_types = {} + for download in failed_downloads: + et = download.error_type or "unknown" + error_types[et] = error_types.get(et, 0) + 1 + + print("\nError type breakdown:") + for et, count in sorted(error_types.items(), key=lambda x: -x[1]): + print(f" {et}: {count}") + + return output_path + + finally: + session.close() + + +def format_download_for_export(download: Download, classification: str) -> dict: + """Format a download record for export.""" + + # Extract logs from metadata + metadata = download.metadata_ or {} + stdout = metadata.get("stdout") or "" + stderr = metadata.get("stderr") or "" + + # Truncate very long logs but keep enough for analysis + max_log_length = 50000 # 50KB per log field + if stdout and len(stdout) > max_log_length: + stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]" + if stderr and len(stderr) > max_log_length: + stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]" + + # Get platform from source if available + platform = None + if download.source: + platform = download.source.platform + + return { + "id": download.id, + "classification": classification, + "assigned_error_type": download.error_type, + "assigned_error_message": download.error_message, + "platform": platform, + "url": download.url, + "status": download.status, # Already a string, not enum + "file_count": download.file_count, + "created_at": download.created_at.isoformat() if download.created_at else None, + "duration_seconds": metadata.get("duration_seconds"), + "logs": { + "stdout": stdout, + "stderr": stderr, + }, + "user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type" + "suggested_error_type": None, # User can suggest what it should be + "notes": None, # User can add context + } + + +def generate_analysis_prompt() -> str: + """Generate a prompt for Claude to analyze the logs.""" + return """ +## Error Log Analysis Task + +I'm providing you with download logs from a gallery-dl based downloader. Each record includes: +- The error_type that was assigned by the current classification logic +- The actual stdout/stderr logs from gallery-dl +- Platform and URL information + +Please analyze these logs and identify: + +1. **False Positives**: Cases where an error was reported but the download actually succeeded + - Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked + +2. **False Negatives**: Cases classified as one error type that should be another + - Example: classified as "not_found" but logs show authentication issues + +3. **Pattern Improvements**: Suggest more specific patterns to detect each error type + - Current patterns may be too broad (matching debug output) or too narrow (missing variations) + +4. **New Error Types**: Any failure modes not currently categorized + +For each issue found, provide: +- The download ID +- Current classification vs suggested classification +- The specific log lines that support your analysis +- Suggested pattern improvements (if applicable) + +Focus on actionable improvements to the _categorize_error() function in gallery_dl.py. +""" + + +def main(): + parser = argparse.ArgumentParser( + description="Export failed download logs for error classification analysis" + ) + parser.add_argument( + "--limit", type=int, default=50, + help="Maximum number of failed records to export (default: 50)" + ) + parser.add_argument( + "--days", type=int, default=30, + help="Only include failures from last N days (default: 30)" + ) + parser.add_argument( + "--error-type", type=str, default=None, + help="Filter by specific error type (e.g., not_found, auth_error)" + ) + parser.add_argument( + "--output", type=str, default="failed_logs_export.json", + help="Output file path (default: failed_logs_export.json)" + ) + parser.add_argument( + "--include-success", action="store_true", + help="Also include some successful 'no new content' records for comparison" + ) + + args = parser.parse_args() + + export_failed_logs( + limit=args.limit, + days=args.days, + error_type=args.error_type, + include_success=args.include_success, + output_path=args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/frontend/src/views/Downloads.vue b/frontend/src/views/Downloads.vue index 9fe4f8e..0df6c02 100644 --- a/frontend/src/views/Downloads.vue +++ b/frontend/src/views/Downloads.vue @@ -55,6 +55,14 @@ Re-queue Stale Queued Jobs Re-send lost Celery tasks for queued jobs + + + + mdi-download + Export Failed Logs + + Download JSON for error analysis + @@ -382,6 +390,18 @@ async function requeueStaleJobs() { } } +async function exportFailedLogs() { + try { + // Open download in new tab - the API returns a file attachment + const baseUrl = import.meta.env.VITE_API_URL || '/api' + const url = `${baseUrl}/downloads/export-failed-logs?limit=100&days=30&include_success=true` + window.open(url, '_blank') + notifications.success('Exporting failed logs...') + } catch (error) { + notifications.error(`Failed to export logs: ${error.message}`) + } +} + function getStatusIcon(status) { const icons = { completed: 'mdi-check-circle', diff --git a/summary.md b/summary.md index 21d411f..4550c7a 100644 --- a/summary.md +++ b/summary.md @@ -1,6 +1,6 @@ # GallerySubscriber - Project Summary -**Updated:** 2026-01-29 (Redis DB config, mismatch detection, stale job recovery, Downloads UI improvements) +**Updated:** 2026-01-30 (Error recognition improvements, analysis tooling) ## Overview @@ -368,3 +368,44 @@ Multiple layers prevent duplicate downloads for the same source: 3. **Scheduler Logic** (`downloads.py`): - `scheduled_check` skips sources with existing QUEUED or RUNNING downloads + +## Error Classification System + +The `_categorize_error()` method in `gallery_dl.py` classifies download failures into actionable error types: + +| Error Type | Meaning | User Action | +|------------|---------|-------------| +| `no_new_content` | All content already downloaded (success) | None needed | +| `auth_error` | Cookies expired or invalid | Re-export cookies from extension | +| `rate_limited` | Too many requests (429) | Increase rate_limit setting | +| `not_found` | Creator/content deleted or moved | Check URL, may need to remove | +| `access_denied` | Insufficient subscription tier | Upgrade pledge or remove source | +| `network_error` | Connection issues | Check network, will auto-retry | +| `timeout` | Download took too long | Will auto-retry | +| `unsupported_url` | URL not recognized by gallery-dl | Check URL format | +| `unknown_error` | Unclassified failure | Check logs for details | + +**Classification Logic:** + +1. **Skip Detection First** - Checks for `#` prefixed lines in stdout (gallery-dl's skip message format) and text patterns like "skipping", "already exists". If skips are detected with no actual errors, returns `no_new_content`. + +2. **Actual Error Detection** - Looks for `][error]` log level, exceptions, and tracebacks before proceeding to pattern matching. + +3. **Specific Patterns** - Uses targeted patterns (e.g., `"timed out"` not just `"timeout"`) to avoid false positives from config values in debug output. + +## Backend Scripts + +| Script | Purpose | +|--------|---------| +| `backend/scripts/export_failed_logs.py` | Export failed download logs to JSON for analysis | + +## Analysis Directory + +The `analysis/` directory (gitignored) contains debugging and analysis artifacts: + +- `failed_logs_export_*.json` - Exported download logs for analysis +- `analysis_results.json` - Parsed analysis results +- `ERROR_RECOGNITION_ANALYSIS.md` - Detailed findings report +- Analysis scripts for pattern debugging + +These files are excluded from version control but preserved locally for ongoing analysis.