tuning error recognition
This commit is contained in:
@@ -14,7 +14,10 @@
|
|||||||
"Bash(where npm:*)",
|
"Bash(where npm:*)",
|
||||||
"Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")",
|
"Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")",
|
||||||
"WebFetch(domain:github.com)",
|
"WebFetch(domain:github.com)",
|
||||||
"WebFetch(domain:raw.githubusercontent.com)"
|
"WebFetch(domain:raw.githubusercontent.com)",
|
||||||
|
"Bash(python -m py_compile:*)",
|
||||||
|
"Bash(powershell:*)",
|
||||||
|
"Bash(python:*)"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -87,3 +87,10 @@ backend/static/
|
|||||||
# Extension packages
|
# Extension packages
|
||||||
*.xpi
|
*.xpi
|
||||||
extension-dist/
|
extension-dist/
|
||||||
|
|
||||||
|
# Analysis and debugging artifacts
|
||||||
|
analysis/
|
||||||
|
**/analysis_*.json
|
||||||
|
**/analysis_*.txt
|
||||||
|
**/failed_logs_export_*.json
|
||||||
|
ERROR_RECOGNITION_ANALYSIS.md
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
"""Download history API endpoints."""
|
"""Download history API endpoints."""
|
||||||
|
|
||||||
from quart import Blueprint, request, jsonify, current_app
|
from datetime import datetime, timedelta
|
||||||
from sqlalchemy import select, func, and_
|
from quart import Blueprint, request, jsonify, current_app, Response
|
||||||
|
from sqlalchemy import select, func, and_, or_, desc
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
|
import json
|
||||||
|
|
||||||
from app.models.base import utcnow
|
from app.models.base import utcnow
|
||||||
from app.models.download import Download, DownloadStatus
|
from app.models.download import Download, DownloadStatus
|
||||||
@@ -285,3 +287,161 @@ async def get_stats():
|
|||||||
"queued": status_counts.get(DownloadStatus.QUEUED, 0),
|
"queued": status_counts.get(DownloadStatus.QUEUED, 0),
|
||||||
"running": status_counts.get(DownloadStatus.RUNNING, 0),
|
"running": status_counts.get(DownloadStatus.RUNNING, 0),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/export-failed-logs", methods=["GET"])
|
||||||
|
async def export_failed_logs():
|
||||||
|
"""Export failed download logs for error classification analysis.
|
||||||
|
|
||||||
|
Returns a JSON file that can be fed to Claude to improve error recognition patterns.
|
||||||
|
|
||||||
|
Query params:
|
||||||
|
limit: Maximum number of records (default 50, max 200)
|
||||||
|
days: Only include failures from last N days (default 30)
|
||||||
|
error_type: Filter by specific error type (e.g., not_found, auth_error)
|
||||||
|
include_success: Also include some successful "no new content" for comparison
|
||||||
|
"""
|
||||||
|
limit = min(int(request.args.get("limit", 50)), 200)
|
||||||
|
days = int(request.args.get("days", 30))
|
||||||
|
error_type = request.args.get("error_type")
|
||||||
|
include_success = request.args.get("include_success", "").lower() in ("true", "1", "yes")
|
||||||
|
|
||||||
|
cutoff_date = utcnow() - timedelta(days=days)
|
||||||
|
|
||||||
|
async with current_app.db_engine.connect() as conn:
|
||||||
|
session = AsyncSession(bind=conn)
|
||||||
|
|
||||||
|
# Build query for failed downloads (eager load source for platform)
|
||||||
|
query = select(Download).options(selectinload(Download.source)).where(
|
||||||
|
and_(
|
||||||
|
Download.status == DownloadStatus.FAILED,
|
||||||
|
Download.created_at >= cutoff_date,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if error_type:
|
||||||
|
query = query.where(Download.error_type == error_type)
|
||||||
|
|
||||||
|
# Order by most recent first
|
||||||
|
query = query.order_by(desc(Download.created_at)).limit(limit)
|
||||||
|
result = await session.execute(query)
|
||||||
|
failed_downloads = result.scalars().all()
|
||||||
|
|
||||||
|
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||||
|
success_downloads = []
|
||||||
|
if include_success:
|
||||||
|
success_query = select(Download).options(selectinload(Download.source)).where(
|
||||||
|
and_(
|
||||||
|
Download.status == DownloadStatus.COMPLETED,
|
||||||
|
Download.created_at >= cutoff_date,
|
||||||
|
Download.file_count == 0, # No new content cases
|
||||||
|
)
|
||||||
|
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||||
|
success_result = await session.execute(success_query)
|
||||||
|
success_downloads = success_result.scalars().all()
|
||||||
|
|
||||||
|
# Format for export
|
||||||
|
exports = []
|
||||||
|
|
||||||
|
for download in failed_downloads:
|
||||||
|
exports.append(_format_download_for_export(download, "failed"))
|
||||||
|
|
||||||
|
for download in success_downloads:
|
||||||
|
exports.append(_format_download_for_export(download, "success_no_new_content"))
|
||||||
|
|
||||||
|
# Build output document
|
||||||
|
output = {
|
||||||
|
"export_info": {
|
||||||
|
"exported_at": utcnow().isoformat(),
|
||||||
|
"total_records": len(exports),
|
||||||
|
"failed_count": len(failed_downloads),
|
||||||
|
"success_comparison_count": len(success_downloads),
|
||||||
|
"days_included": days,
|
||||||
|
"error_type_filter": error_type,
|
||||||
|
},
|
||||||
|
"analysis_prompt": _generate_analysis_prompt(),
|
||||||
|
"downloads": exports,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Return as downloadable JSON file
|
||||||
|
response = Response(
|
||||||
|
json.dumps(output, indent=2, default=str),
|
||||||
|
mimetype="application/json",
|
||||||
|
headers={
|
||||||
|
"Content-Disposition": f"attachment; filename=failed_logs_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def _format_download_for_export(download: Download, classification: str) -> dict:
|
||||||
|
"""Format a download record for export."""
|
||||||
|
# Extract logs from metadata
|
||||||
|
metadata = download.metadata_ or {}
|
||||||
|
stdout = metadata.get("stdout") or ""
|
||||||
|
stderr = metadata.get("stderr") or ""
|
||||||
|
|
||||||
|
# Truncate very long logs but keep enough for analysis
|
||||||
|
max_log_length = 50000 # 50KB per log field
|
||||||
|
if stdout and len(stdout) > max_log_length:
|
||||||
|
stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]"
|
||||||
|
if stderr and len(stderr) > max_log_length:
|
||||||
|
stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]"
|
||||||
|
|
||||||
|
# Get platform from source if available
|
||||||
|
platform = None
|
||||||
|
if download.source:
|
||||||
|
platform = download.source.platform
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": download.id,
|
||||||
|
"classification": classification,
|
||||||
|
"assigned_error_type": download.error_type,
|
||||||
|
"assigned_error_message": download.error_message,
|
||||||
|
"platform": platform,
|
||||||
|
"url": download.url,
|
||||||
|
"status": download.status, # Already a string, not enum
|
||||||
|
"file_count": download.file_count,
|
||||||
|
"created_at": download.created_at.isoformat() if download.created_at else None,
|
||||||
|
"duration_seconds": metadata.get("duration_seconds"),
|
||||||
|
"logs": {
|
||||||
|
"stdout": stdout,
|
||||||
|
"stderr": stderr,
|
||||||
|
},
|
||||||
|
"user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type"
|
||||||
|
"suggested_error_type": None, # User can suggest what it should be
|
||||||
|
"notes": None, # User can add context
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _generate_analysis_prompt() -> str:
|
||||||
|
"""Generate a prompt for Claude to analyze the logs."""
|
||||||
|
return """
|
||||||
|
## Error Log Analysis Task
|
||||||
|
|
||||||
|
I'm providing you with download logs from a gallery-dl based downloader. Each record includes:
|
||||||
|
- The error_type that was assigned by the current classification logic
|
||||||
|
- The actual stdout/stderr logs from gallery-dl
|
||||||
|
- Platform and URL information
|
||||||
|
|
||||||
|
Please analyze these logs and identify:
|
||||||
|
|
||||||
|
1. **False Positives**: Cases where an error was reported but the download actually succeeded
|
||||||
|
- Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked
|
||||||
|
|
||||||
|
2. **False Negatives**: Cases classified as one error type that should be another
|
||||||
|
- Example: classified as "not_found" but logs show authentication issues
|
||||||
|
|
||||||
|
3. **Pattern Improvements**: Suggest more specific patterns to detect each error type
|
||||||
|
- Current patterns may be too broad (matching debug output) or too narrow (missing variations)
|
||||||
|
|
||||||
|
4. **New Error Types**: Any failure modes not currently categorized
|
||||||
|
|
||||||
|
For each issue found, provide:
|
||||||
|
- The download ID
|
||||||
|
- Current classification vs suggested classification
|
||||||
|
- The specific log lines that support your analysis
|
||||||
|
- Suggested pattern improvements (if applicable)
|
||||||
|
|
||||||
|
Focus on actionable improvements to the _categorize_error() function in gallery_dl.py.
|
||||||
|
"""
|
||||||
|
|||||||
@@ -316,23 +316,52 @@ class GalleryDLService:
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]:
|
def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]:
|
||||||
"""Categorize the error based on output and return code."""
|
"""Categorize the error based on output and return code.
|
||||||
|
|
||||||
|
Priority order:
|
||||||
|
1. Check for skip messages (# prefixed lines or text patterns) → NO_NEW_CONTENT
|
||||||
|
2. Check for actual errors (auth, rate limit, 404, network, etc.)
|
||||||
|
3. Fall back to UNKNOWN_ERROR
|
||||||
|
|
||||||
|
This ordering prevents false positives where debug output matches error patterns.
|
||||||
|
"""
|
||||||
combined = f"{stdout} {stderr}".lower()
|
combined = f"{stdout} {stderr}".lower()
|
||||||
|
|
||||||
# Check for "no new content" FIRST for any non-zero return code
|
# === PHASE 1: Detect skip/no-new-content scenarios ===
|
||||||
# Gallery-dl returns 1 when all files are skipped or no new content found
|
# Gallery-dl indicates skipped files two ways:
|
||||||
# This check must run before other error checks to avoid false positives
|
# 1. Lines starting with '#' in stdout (e.g., "# /data/downloads/Artist/file.png")
|
||||||
skip_indicators = ["skipping", "skip", "already exists", "archive"]
|
# 2. Text patterns like "skipping", "already exists"
|
||||||
if any(indicator in combined for indicator in skip_indicators):
|
|
||||||
# If we see skip messages and no actual error messages, it's just "no new content"
|
# Count lines starting with '#' (skip messages)
|
||||||
actual_errors = ["error", "failed", "exception", "traceback"]
|
skip_line_count = len([
|
||||||
if not any(err in combined for err in actual_errors):
|
line for line in stdout.split('\n')
|
||||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
if line.strip().startswith('#')
|
||||||
|
])
|
||||||
|
|
||||||
|
# Text-based skip indicators
|
||||||
|
skip_text_indicators = ["skipping", "already exists", "archive"]
|
||||||
|
has_skip_text = any(ind in combined for ind in skip_text_indicators)
|
||||||
|
|
||||||
|
# Actual error indicators that override skip detection
|
||||||
|
actual_error_indicators = [
|
||||||
|
"][error]", # gallery-dl error log level: [patreon][error]
|
||||||
|
"exception", # Python exceptions
|
||||||
|
"traceback", # Python tracebacks
|
||||||
|
"download failed",
|
||||||
|
"extraction failed",
|
||||||
|
]
|
||||||
|
has_actual_error = any(err in combined for err in actual_error_indicators)
|
||||||
|
|
||||||
|
# If we have skip evidence and no real errors, it's no_new_content
|
||||||
|
if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||||
|
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||||
|
|
||||||
# Empty output with low return codes often means nothing to download
|
# Empty output with low return codes often means nothing to download
|
||||||
if return_code in (0, 1) and not stdout.strip():
|
if return_code in (0, 1) and not stdout.strip():
|
||||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||||
|
|
||||||
|
# === PHASE 2: Check for specific error types ===
|
||||||
|
|
||||||
# Auth errors - look for error context, not just HTTP codes that appear in debug logs
|
# Auth errors - look for error context, not just HTTP codes that appear in debug logs
|
||||||
# HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
|
# HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
|
||||||
auth_error_patterns = [
|
auth_error_patterns = [
|
||||||
@@ -370,29 +399,50 @@ class GalleryDLService:
|
|||||||
if any(pattern in combined for pattern in rate_limit_patterns):
|
if any(pattern in combined for pattern in rate_limit_patterns):
|
||||||
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
|
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
|
||||||
|
|
||||||
# 404 Not Found - check for HTTP 404 responses, not just "404" anywhere
|
# 404 Not Found - check for HTTP 404 responses with error context
|
||||||
|
# Be specific to avoid matching "not found" in debug messages like "archive entry not found"
|
||||||
not_found_patterns = [
|
not_found_patterns = [
|
||||||
'" 404 ', # HTTP response
|
'" 404 ', # HTTP response log line
|
||||||
"error 404",
|
"error 404", # Explicit error 404
|
||||||
"status: 404",
|
"status: 404", # Status code indicator
|
||||||
"not found",
|
"status code: 404", # Status code indicator
|
||||||
"does not exist",
|
"404 not found", # Standard 404 message
|
||||||
"no longer available",
|
"page not found", # Common 404 page text
|
||||||
|
"user not found", # User doesn't exist
|
||||||
|
"creator not found",
|
||||||
|
"artist not found",
|
||||||
|
"content no longer available",
|
||||||
|
"has been deleted",
|
||||||
|
"account.*deleted",
|
||||||
|
"profile.*not.*exist",
|
||||||
]
|
]
|
||||||
if any(pattern in combined for pattern in not_found_patterns):
|
# Also require error-level context for generic patterns to avoid false positives
|
||||||
|
generic_not_found = ["does not exist", "no longer available"]
|
||||||
|
has_specific_404 = any(pattern in combined for pattern in not_found_patterns)
|
||||||
|
has_generic_with_error = any(
|
||||||
|
pattern in combined and "][error]" in combined
|
||||||
|
for pattern in generic_not_found
|
||||||
|
)
|
||||||
|
if has_specific_404 or has_generic_with_error:
|
||||||
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
|
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
|
||||||
|
|
||||||
# Network errors
|
# Network errors - use specific patterns to avoid matching config values
|
||||||
|
# (e.g., "timeout": 30.0 in debug output should not trigger network_error)
|
||||||
network_error_patterns = [
|
network_error_patterns = [
|
||||||
"timeout",
|
"timed out", # Actual timeout occurred
|
||||||
"timed out",
|
"read timed out", # urllib3/requests timeout
|
||||||
|
"connect timed out", # Connection timeout
|
||||||
|
"connection timed out",
|
||||||
"connection refused",
|
"connection refused",
|
||||||
"connection reset",
|
"connection reset",
|
||||||
"network unreachable",
|
"network unreachable",
|
||||||
"name resolution",
|
"name resolution failed",
|
||||||
"dns",
|
"name or service not known",
|
||||||
"ssl error",
|
"nodename nor servname provided",
|
||||||
"certificate verify",
|
"ssl: certificate_verify_failed",
|
||||||
|
"ssl: wrong_version_number",
|
||||||
|
"certificate verify failed",
|
||||||
|
"[errno", # Socket errors like [Errno 111]
|
||||||
]
|
]
|
||||||
if any(pattern in combined for pattern in network_error_patterns):
|
if any(pattern in combined for pattern in network_error_patterns):
|
||||||
return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
|
return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
|
||||||
@@ -418,10 +468,12 @@ class GalleryDLService:
|
|||||||
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
||||||
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
||||||
|
|
||||||
# If we got here with return code 1 and saw some activity, it might just be no new content
|
# If we got here with return code 1 and saw some activity but no error patterns matched,
|
||||||
if return_code == 1:
|
# it's likely just no new content (gallery-dl returns 1 when nothing to download)
|
||||||
|
if return_code == 1 and stdout.strip() and not has_actual_error:
|
||||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||||
|
|
||||||
|
# Final fallback - genuine unknown error
|
||||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||||
|
|
||||||
def _count_downloaded_files(self, stdout: str) -> int:
|
def _count_downloaded_files(self, stdout: str) -> int:
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Export failed download logs for error classification analysis.
|
||||||
|
|
||||||
|
Outputs a JSON file that can be fed to Claude to improve error recognition patterns.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python scripts/export_failed_logs.py [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--limit N Maximum number of records to export (default: 50)
|
||||||
|
--days N Only include failures from last N days (default: 30)
|
||||||
|
--error-type TYPE Filter by specific error type (e.g., not_found, auth_error)
|
||||||
|
--output FILE Output file path (default: failed_logs_export.json)
|
||||||
|
--include-success Also include some successful "no new content" for comparison
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Add parent directory to path for imports
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, desc, or_
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from app.models.download import Download, DownloadStatus
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
|
||||||
|
def export_failed_logs(
|
||||||
|
limit: int = 50,
|
||||||
|
days: int = 30,
|
||||||
|
error_type: str | None = None,
|
||||||
|
include_success: bool = False,
|
||||||
|
output_path: str = "failed_logs_export.json",
|
||||||
|
):
|
||||||
|
"""Export failed download logs to JSON for analysis."""
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
Session = sessionmaker(bind=engine)
|
||||||
|
session = Session()
|
||||||
|
|
||||||
|
try:
|
||||||
|
cutoff_date = datetime.utcnow() - timedelta(days=days)
|
||||||
|
|
||||||
|
# Build query for failed downloads
|
||||||
|
query = session.query(Download).filter(
|
||||||
|
Download.status == DownloadStatus.FAILED,
|
||||||
|
Download.created_at >= cutoff_date,
|
||||||
|
)
|
||||||
|
|
||||||
|
if error_type:
|
||||||
|
query = query.filter(Download.error_type == error_type)
|
||||||
|
|
||||||
|
# Order by most recent first
|
||||||
|
failed_downloads = query.order_by(desc(Download.created_at)).limit(limit).all()
|
||||||
|
|
||||||
|
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||||
|
success_downloads = []
|
||||||
|
if include_success:
|
||||||
|
success_query = session.query(Download).filter(
|
||||||
|
Download.status == DownloadStatus.COMPLETED,
|
||||||
|
Download.created_at >= cutoff_date,
|
||||||
|
Download.file_count == 0, # No new content cases
|
||||||
|
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||||
|
success_downloads = success_query.all()
|
||||||
|
|
||||||
|
# Format for export
|
||||||
|
exports = []
|
||||||
|
|
||||||
|
for download in failed_downloads:
|
||||||
|
exports.append(format_download_for_export(download, "failed"))
|
||||||
|
|
||||||
|
for download in success_downloads:
|
||||||
|
exports.append(format_download_for_export(download, "success_no_new_content"))
|
||||||
|
|
||||||
|
# Build output document
|
||||||
|
output = {
|
||||||
|
"export_info": {
|
||||||
|
"exported_at": datetime.utcnow().isoformat(),
|
||||||
|
"total_records": len(exports),
|
||||||
|
"failed_count": len(failed_downloads),
|
||||||
|
"success_comparison_count": len(success_downloads),
|
||||||
|
"days_included": days,
|
||||||
|
"error_type_filter": error_type,
|
||||||
|
},
|
||||||
|
"analysis_prompt": generate_analysis_prompt(),
|
||||||
|
"downloads": exports,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write to file
|
||||||
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(output, f, indent=2, default=str)
|
||||||
|
|
||||||
|
print(f"Exported {len(exports)} records to {output_path}")
|
||||||
|
print(f" - Failed: {len(failed_downloads)}")
|
||||||
|
if include_success:
|
||||||
|
print(f" - Success (no new content): {len(success_downloads)}")
|
||||||
|
|
||||||
|
# Print summary by error type
|
||||||
|
error_types = {}
|
||||||
|
for download in failed_downloads:
|
||||||
|
et = download.error_type or "unknown"
|
||||||
|
error_types[et] = error_types.get(et, 0) + 1
|
||||||
|
|
||||||
|
print("\nError type breakdown:")
|
||||||
|
for et, count in sorted(error_types.items(), key=lambda x: -x[1]):
|
||||||
|
print(f" {et}: {count}")
|
||||||
|
|
||||||
|
return output_path
|
||||||
|
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def format_download_for_export(download: Download, classification: str) -> dict:
|
||||||
|
"""Format a download record for export."""
|
||||||
|
|
||||||
|
# Extract logs from metadata
|
||||||
|
metadata = download.metadata_ or {}
|
||||||
|
stdout = metadata.get("stdout") or ""
|
||||||
|
stderr = metadata.get("stderr") or ""
|
||||||
|
|
||||||
|
# Truncate very long logs but keep enough for analysis
|
||||||
|
max_log_length = 50000 # 50KB per log field
|
||||||
|
if stdout and len(stdout) > max_log_length:
|
||||||
|
stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]"
|
||||||
|
if stderr and len(stderr) > max_log_length:
|
||||||
|
stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]"
|
||||||
|
|
||||||
|
# Get platform from source if available
|
||||||
|
platform = None
|
||||||
|
if download.source:
|
||||||
|
platform = download.source.platform
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": download.id,
|
||||||
|
"classification": classification,
|
||||||
|
"assigned_error_type": download.error_type,
|
||||||
|
"assigned_error_message": download.error_message,
|
||||||
|
"platform": platform,
|
||||||
|
"url": download.url,
|
||||||
|
"status": download.status, # Already a string, not enum
|
||||||
|
"file_count": download.file_count,
|
||||||
|
"created_at": download.created_at.isoformat() if download.created_at else None,
|
||||||
|
"duration_seconds": metadata.get("duration_seconds"),
|
||||||
|
"logs": {
|
||||||
|
"stdout": stdout,
|
||||||
|
"stderr": stderr,
|
||||||
|
},
|
||||||
|
"user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type"
|
||||||
|
"suggested_error_type": None, # User can suggest what it should be
|
||||||
|
"notes": None, # User can add context
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_analysis_prompt() -> str:
|
||||||
|
"""Generate a prompt for Claude to analyze the logs."""
|
||||||
|
return """
|
||||||
|
## Error Log Analysis Task
|
||||||
|
|
||||||
|
I'm providing you with download logs from a gallery-dl based downloader. Each record includes:
|
||||||
|
- The error_type that was assigned by the current classification logic
|
||||||
|
- The actual stdout/stderr logs from gallery-dl
|
||||||
|
- Platform and URL information
|
||||||
|
|
||||||
|
Please analyze these logs and identify:
|
||||||
|
|
||||||
|
1. **False Positives**: Cases where an error was reported but the download actually succeeded
|
||||||
|
- Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked
|
||||||
|
|
||||||
|
2. **False Negatives**: Cases classified as one error type that should be another
|
||||||
|
- Example: classified as "not_found" but logs show authentication issues
|
||||||
|
|
||||||
|
3. **Pattern Improvements**: Suggest more specific patterns to detect each error type
|
||||||
|
- Current patterns may be too broad (matching debug output) or too narrow (missing variations)
|
||||||
|
|
||||||
|
4. **New Error Types**: Any failure modes not currently categorized
|
||||||
|
|
||||||
|
For each issue found, provide:
|
||||||
|
- The download ID
|
||||||
|
- Current classification vs suggested classification
|
||||||
|
- The specific log lines that support your analysis
|
||||||
|
- Suggested pattern improvements (if applicable)
|
||||||
|
|
||||||
|
Focus on actionable improvements to the _categorize_error() function in gallery_dl.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Export failed download logs for error classification analysis"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--limit", type=int, default=50,
|
||||||
|
help="Maximum number of failed records to export (default: 50)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--days", type=int, default=30,
|
||||||
|
help="Only include failures from last N days (default: 30)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--error-type", type=str, default=None,
|
||||||
|
help="Filter by specific error type (e.g., not_found, auth_error)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output", type=str, default="failed_logs_export.json",
|
||||||
|
help="Output file path (default: failed_logs_export.json)"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--include-success", action="store_true",
|
||||||
|
help="Also include some successful 'no new content' records for comparison"
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
export_failed_logs(
|
||||||
|
limit=args.limit,
|
||||||
|
days=args.days,
|
||||||
|
error_type=args.error_type,
|
||||||
|
include_success=args.include_success,
|
||||||
|
output_path=args.output,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -55,6 +55,14 @@
|
|||||||
<v-list-item-title>Re-queue Stale Queued Jobs</v-list-item-title>
|
<v-list-item-title>Re-queue Stale Queued Jobs</v-list-item-title>
|
||||||
<v-list-item-subtitle>Re-send lost Celery tasks for queued jobs</v-list-item-subtitle>
|
<v-list-item-subtitle>Re-send lost Celery tasks for queued jobs</v-list-item-subtitle>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
|
<v-divider />
|
||||||
|
<v-list-item @click="exportFailedLogs">
|
||||||
|
<v-list-item-title>
|
||||||
|
<v-icon start size="small">mdi-download</v-icon>
|
||||||
|
Export Failed Logs
|
||||||
|
</v-list-item-title>
|
||||||
|
<v-list-item-subtitle>Download JSON for error analysis</v-list-item-subtitle>
|
||||||
|
</v-list-item>
|
||||||
</v-list>
|
</v-list>
|
||||||
</v-menu>
|
</v-menu>
|
||||||
</v-col>
|
</v-col>
|
||||||
@@ -382,6 +390,18 @@ async function requeueStaleJobs() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function exportFailedLogs() {
|
||||||
|
try {
|
||||||
|
// Open download in new tab - the API returns a file attachment
|
||||||
|
const baseUrl = import.meta.env.VITE_API_URL || '/api'
|
||||||
|
const url = `${baseUrl}/downloads/export-failed-logs?limit=100&days=30&include_success=true`
|
||||||
|
window.open(url, '_blank')
|
||||||
|
notifications.success('Exporting failed logs...')
|
||||||
|
} catch (error) {
|
||||||
|
notifications.error(`Failed to export logs: ${error.message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getStatusIcon(status) {
|
function getStatusIcon(status) {
|
||||||
const icons = {
|
const icons = {
|
||||||
completed: 'mdi-check-circle',
|
completed: 'mdi-check-circle',
|
||||||
|
|||||||
+42
-1
@@ -1,6 +1,6 @@
|
|||||||
# GallerySubscriber - Project Summary
|
# GallerySubscriber - Project Summary
|
||||||
|
|
||||||
**Updated:** 2026-01-29 (Redis DB config, mismatch detection, stale job recovery, Downloads UI improvements)
|
**Updated:** 2026-01-30 (Error recognition improvements, analysis tooling)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
@@ -368,3 +368,44 @@ Multiple layers prevent duplicate downloads for the same source:
|
|||||||
|
|
||||||
3. **Scheduler Logic** (`downloads.py`):
|
3. **Scheduler Logic** (`downloads.py`):
|
||||||
- `scheduled_check` skips sources with existing QUEUED or RUNNING downloads
|
- `scheduled_check` skips sources with existing QUEUED or RUNNING downloads
|
||||||
|
|
||||||
|
## Error Classification System
|
||||||
|
|
||||||
|
The `_categorize_error()` method in `gallery_dl.py` classifies download failures into actionable error types:
|
||||||
|
|
||||||
|
| Error Type | Meaning | User Action |
|
||||||
|
|------------|---------|-------------|
|
||||||
|
| `no_new_content` | All content already downloaded (success) | None needed |
|
||||||
|
| `auth_error` | Cookies expired or invalid | Re-export cookies from extension |
|
||||||
|
| `rate_limited` | Too many requests (429) | Increase rate_limit setting |
|
||||||
|
| `not_found` | Creator/content deleted or moved | Check URL, may need to remove |
|
||||||
|
| `access_denied` | Insufficient subscription tier | Upgrade pledge or remove source |
|
||||||
|
| `network_error` | Connection issues | Check network, will auto-retry |
|
||||||
|
| `timeout` | Download took too long | Will auto-retry |
|
||||||
|
| `unsupported_url` | URL not recognized by gallery-dl | Check URL format |
|
||||||
|
| `unknown_error` | Unclassified failure | Check logs for details |
|
||||||
|
|
||||||
|
**Classification Logic:**
|
||||||
|
|
||||||
|
1. **Skip Detection First** - Checks for `#` prefixed lines in stdout (gallery-dl's skip message format) and text patterns like "skipping", "already exists". If skips are detected with no actual errors, returns `no_new_content`.
|
||||||
|
|
||||||
|
2. **Actual Error Detection** - Looks for `][error]` log level, exceptions, and tracebacks before proceeding to pattern matching.
|
||||||
|
|
||||||
|
3. **Specific Patterns** - Uses targeted patterns (e.g., `"timed out"` not just `"timeout"`) to avoid false positives from config values in debug output.
|
||||||
|
|
||||||
|
## Backend Scripts
|
||||||
|
|
||||||
|
| Script | Purpose |
|
||||||
|
|--------|---------|
|
||||||
|
| `backend/scripts/export_failed_logs.py` | Export failed download logs to JSON for analysis |
|
||||||
|
|
||||||
|
## Analysis Directory
|
||||||
|
|
||||||
|
The `analysis/` directory (gitignored) contains debugging and analysis artifacts:
|
||||||
|
|
||||||
|
- `failed_logs_export_*.json` - Exported download logs for analysis
|
||||||
|
- `analysis_results.json` - Parsed analysis results
|
||||||
|
- `ERROR_RECOGNITION_ANALYSIS.md` - Detailed findings report
|
||||||
|
- Analysis scripts for pattern debugging
|
||||||
|
|
||||||
|
These files are excluded from version control but preserved locally for ongoing analysis.
|
||||||
|
|||||||
Reference in New Issue
Block a user