tuning error recognition
This commit is contained in:
Bryan Van Deusen
@@ -14,7 +14,10 @@
|
||||
"Bash(where npm:*)",
|
||||
"Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")",
|
||||
"WebFetch(domain:github.com)",
|
||||
"WebFetch(domain:raw.githubusercontent.com)"
|
||||
"WebFetch(domain:raw.githubusercontent.com)",
|
||||
"Bash(python -m py_compile:*)",
|
||||
"Bash(powershell:*)",
|
||||
"Bash(python:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,3 +87,10 @@ backend/static/
|
||||
# Extension packages
|
||||
*.xpi
|
||||
extension-dist/
|
||||
|
||||
# Analysis and debugging artifacts
|
||||
analysis/
|
||||
**/analysis_*.json
|
||||
**/analysis_*.txt
|
||||
**/failed_logs_export_*.json
|
||||
ERROR_RECOGNITION_ANALYSIS.md
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Download history API endpoints."""
|
||||
|
||||
from quart import Blueprint, request, jsonify, current_app
|
||||
from sqlalchemy import select, func, and_
|
||||
from datetime import datetime, timedelta
|
||||
from quart import Blueprint, request, jsonify, current_app, Response
|
||||
from sqlalchemy import select, func, and_, or_, desc
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
import json
|
||||
|
||||
from app.models.base import utcnow
|
||||
from app.models.download import Download, DownloadStatus
|
||||
@@ -285,3 +287,161 @@ async def get_stats():
|
||||
"queued": status_counts.get(DownloadStatus.QUEUED, 0),
|
||||
"running": status_counts.get(DownloadStatus.RUNNING, 0),
|
||||
})
|
||||
|
||||
|
||||
@bp.route("/export-failed-logs", methods=["GET"])
|
||||
async def export_failed_logs():
|
||||
"""Export failed download logs for error classification analysis.
|
||||
|
||||
Returns a JSON file that can be fed to Claude to improve error recognition patterns.
|
||||
|
||||
Query params:
|
||||
limit: Maximum number of records (default 50, max 200)
|
||||
days: Only include failures from last N days (default 30)
|
||||
error_type: Filter by specific error type (e.g., not_found, auth_error)
|
||||
include_success: Also include some successful "no new content" for comparison
|
||||
"""
|
||||
limit = min(int(request.args.get("limit", 50)), 200)
|
||||
days = int(request.args.get("days", 30))
|
||||
error_type = request.args.get("error_type")
|
||||
include_success = request.args.get("include_success", "").lower() in ("true", "1", "yes")
|
||||
|
||||
cutoff_date = utcnow() - timedelta(days=days)
|
||||
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
|
||||
# Build query for failed downloads (eager load source for platform)
|
||||
query = select(Download).options(selectinload(Download.source)).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.created_at >= cutoff_date,
|
||||
)
|
||||
)
|
||||
|
||||
if error_type:
|
||||
query = query.where(Download.error_type == error_type)
|
||||
|
||||
# Order by most recent first
|
||||
query = query.order_by(desc(Download.created_at)).limit(limit)
|
||||
result = await session.execute(query)
|
||||
failed_downloads = result.scalars().all()
|
||||
|
||||
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||
success_downloads = []
|
||||
if include_success:
|
||||
success_query = select(Download).options(selectinload(Download.source)).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.created_at >= cutoff_date,
|
||||
Download.file_count == 0, # No new content cases
|
||||
)
|
||||
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||
success_result = await session.execute(success_query)
|
||||
success_downloads = success_result.scalars().all()
|
||||
|
||||
# Format for export
|
||||
exports = []
|
||||
|
||||
for download in failed_downloads:
|
||||
exports.append(_format_download_for_export(download, "failed"))
|
||||
|
||||
for download in success_downloads:
|
||||
exports.append(_format_download_for_export(download, "success_no_new_content"))
|
||||
|
||||
# Build output document
|
||||
output = {
|
||||
"export_info": {
|
||||
"exported_at": utcnow().isoformat(),
|
||||
"total_records": len(exports),
|
||||
"failed_count": len(failed_downloads),
|
||||
"success_comparison_count": len(success_downloads),
|
||||
"days_included": days,
|
||||
"error_type_filter": error_type,
|
||||
},
|
||||
"analysis_prompt": _generate_analysis_prompt(),
|
||||
"downloads": exports,
|
||||
}
|
||||
|
||||
# Return as downloadable JSON file
|
||||
response = Response(
|
||||
json.dumps(output, indent=2, default=str),
|
||||
mimetype="application/json",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=failed_logs_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||||
}
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _format_download_for_export(download: Download, classification: str) -> dict:
|
||||
"""Format a download record for export."""
|
||||
# Extract logs from metadata
|
||||
metadata = download.metadata_ or {}
|
||||
stdout = metadata.get("stdout") or ""
|
||||
stderr = metadata.get("stderr") or ""
|
||||
|
||||
# Truncate very long logs but keep enough for analysis
|
||||
max_log_length = 50000 # 50KB per log field
|
||||
if stdout and len(stdout) > max_log_length:
|
||||
stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]"
|
||||
if stderr and len(stderr) > max_log_length:
|
||||
stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]"
|
||||
|
||||
# Get platform from source if available
|
||||
platform = None
|
||||
if download.source:
|
||||
platform = download.source.platform
|
||||
|
||||
return {
|
||||
"id": download.id,
|
||||
"classification": classification,
|
||||
"assigned_error_type": download.error_type,
|
||||
"assigned_error_message": download.error_message,
|
||||
"platform": platform,
|
||||
"url": download.url,
|
||||
"status": download.status, # Already a string, not enum
|
||||
"file_count": download.file_count,
|
||||
"created_at": download.created_at.isoformat() if download.created_at else None,
|
||||
"duration_seconds": metadata.get("duration_seconds"),
|
||||
"logs": {
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
},
|
||||
"user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type"
|
||||
"suggested_error_type": None, # User can suggest what it should be
|
||||
"notes": None, # User can add context
|
||||
}
|
||||
|
||||
|
||||
def _generate_analysis_prompt() -> str:
|
||||
"""Generate a prompt for Claude to analyze the logs."""
|
||||
return """
|
||||
## Error Log Analysis Task
|
||||
|
||||
I'm providing you with download logs from a gallery-dl based downloader. Each record includes:
|
||||
- The error_type that was assigned by the current classification logic
|
||||
- The actual stdout/stderr logs from gallery-dl
|
||||
- Platform and URL information
|
||||
|
||||
Please analyze these logs and identify:
|
||||
|
||||
1. **False Positives**: Cases where an error was reported but the download actually succeeded
|
||||
- Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked
|
||||
|
||||
2. **False Negatives**: Cases classified as one error type that should be another
|
||||
- Example: classified as "not_found" but logs show authentication issues
|
||||
|
||||
3. **Pattern Improvements**: Suggest more specific patterns to detect each error type
|
||||
- Current patterns may be too broad (matching debug output) or too narrow (missing variations)
|
||||
|
||||
4. **New Error Types**: Any failure modes not currently categorized
|
||||
|
||||
For each issue found, provide:
|
||||
- The download ID
|
||||
- Current classification vs suggested classification
|
||||
- The specific log lines that support your analysis
|
||||
- Suggested pattern improvements (if applicable)
|
||||
|
||||
Focus on actionable improvements to the _categorize_error() function in gallery_dl.py.
|
||||
"""
|
||||
|
||||
@@ -316,23 +316,52 @@ class GalleryDLService:
|
||||
return config
|
||||
|
||||
def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]:
|
||||
"""Categorize the error based on output and return code."""
|
||||
"""Categorize the error based on output and return code.
|
||||
|
||||
Priority order:
|
||||
1. Check for skip messages (# prefixed lines or text patterns) → NO_NEW_CONTENT
|
||||
2. Check for actual errors (auth, rate limit, 404, network, etc.)
|
||||
3. Fall back to UNKNOWN_ERROR
|
||||
|
||||
This ordering prevents false positives where debug output matches error patterns.
|
||||
"""
|
||||
combined = f"{stdout} {stderr}".lower()
|
||||
|
||||
# Check for "no new content" FIRST for any non-zero return code
|
||||
# Gallery-dl returns 1 when all files are skipped or no new content found
|
||||
# This check must run before other error checks to avoid false positives
|
||||
skip_indicators = ["skipping", "skip", "already exists", "archive"]
|
||||
if any(indicator in combined for indicator in skip_indicators):
|
||||
# If we see skip messages and no actual error messages, it's just "no new content"
|
||||
actual_errors = ["error", "failed", "exception", "traceback"]
|
||||
if not any(err in combined for err in actual_errors):
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
# === PHASE 1: Detect skip/no-new-content scenarios ===
|
||||
# Gallery-dl indicates skipped files two ways:
|
||||
# 1. Lines starting with '#' in stdout (e.g., "# /data/downloads/Artist/file.png")
|
||||
# 2. Text patterns like "skipping", "already exists"
|
||||
|
||||
# Count lines starting with '#' (skip messages)
|
||||
skip_line_count = len([
|
||||
line for line in stdout.split('\n')
|
||||
if line.strip().startswith('#')
|
||||
])
|
||||
|
||||
# Text-based skip indicators
|
||||
skip_text_indicators = ["skipping", "already exists", "archive"]
|
||||
has_skip_text = any(ind in combined for ind in skip_text_indicators)
|
||||
|
||||
# Actual error indicators that override skip detection
|
||||
actual_error_indicators = [
|
||||
"][error]", # gallery-dl error log level: [patreon][error]
|
||||
"exception", # Python exceptions
|
||||
"traceback", # Python tracebacks
|
||||
"download failed",
|
||||
"extraction failed",
|
||||
]
|
||||
has_actual_error = any(err in combined for err in actual_error_indicators)
|
||||
|
||||
# If we have skip evidence and no real errors, it's no_new_content
|
||||
if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
# Empty output with low return codes often means nothing to download
|
||||
if return_code in (0, 1) and not stdout.strip():
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
# === PHASE 2: Check for specific error types ===
|
||||
|
||||
# Auth errors - look for error context, not just HTTP codes that appear in debug logs
|
||||
# HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
|
||||
auth_error_patterns = [
|
||||
@@ -370,29 +399,50 @@ class GalleryDLService:
|
||||
if any(pattern in combined for pattern in rate_limit_patterns):
|
||||
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
|
||||
|
||||
# 404 Not Found - check for HTTP 404 responses, not just "404" anywhere
|
||||
# 404 Not Found - check for HTTP 404 responses with error context
|
||||
# Be specific to avoid matching "not found" in debug messages like "archive entry not found"
|
||||
not_found_patterns = [
|
||||
'" 404 ', # HTTP response
|
||||
"error 404",
|
||||
"status: 404",
|
||||
"not found",
|
||||
"does not exist",
|
||||
"no longer available",
|
||||
'" 404 ', # HTTP response log line
|
||||
"error 404", # Explicit error 404
|
||||
"status: 404", # Status code indicator
|
||||
"status code: 404", # Status code indicator
|
||||
"404 not found", # Standard 404 message
|
||||
"page not found", # Common 404 page text
|
||||
"user not found", # User doesn't exist
|
||||
"creator not found",
|
||||
"artist not found",
|
||||
"content no longer available",
|
||||
"has been deleted",
|
||||
"account.*deleted",
|
||||
"profile.*not.*exist",
|
||||
]
|
||||
if any(pattern in combined for pattern in not_found_patterns):
|
||||
# Also require error-level context for generic patterns to avoid false positives
|
||||
generic_not_found = ["does not exist", "no longer available"]
|
||||
has_specific_404 = any(pattern in combined for pattern in not_found_patterns)
|
||||
has_generic_with_error = any(
|
||||
pattern in combined and "][error]" in combined
|
||||
for pattern in generic_not_found
|
||||
)
|
||||
if has_specific_404 or has_generic_with_error:
|
||||
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
|
||||
|
||||
# Network errors
|
||||
# Network errors - use specific patterns to avoid matching config values
|
||||
# (e.g., "timeout": 30.0 in debug output should not trigger network_error)
|
||||
network_error_patterns = [
|
||||
"timeout",
|
||||
"timed out",
|
||||
"timed out", # Actual timeout occurred
|
||||
"read timed out", # urllib3/requests timeout
|
||||
"connect timed out", # Connection timeout
|
||||
"connection timed out",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"network unreachable",
|
||||
"name resolution",
|
||||
"dns",
|
||||
"ssl error",
|
||||
"certificate verify",
|
||||
"name resolution failed",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"ssl: certificate_verify_failed",
|
||||
"ssl: wrong_version_number",
|
||||
"certificate verify failed",
|
||||
"[errno", # Socket errors like [Errno 111]
|
||||
]
|
||||
if any(pattern in combined for pattern in network_error_patterns):
|
||||
return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
|
||||
@@ -418,10 +468,12 @@ class GalleryDLService:
|
||||
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
||||
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
||||
|
||||
# If we got here with return code 1 and saw some activity, it might just be no new content
|
||||
if return_code == 1:
|
||||
# If we got here with return code 1 and saw some activity but no error patterns matched,
|
||||
# it's likely just no new content (gallery-dl returns 1 when nothing to download)
|
||||
if return_code == 1 and stdout.strip() and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
# Final fallback - genuine unknown error
|
||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||
|
||||
def _count_downloaded_files(self, stdout: str) -> int:
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Export failed download logs for error classification analysis.
|
||||
|
||||
Outputs a JSON file that can be fed to Claude to improve error recognition patterns.
|
||||
|
||||
Usage:
|
||||
python scripts/export_failed_logs.py [options]
|
||||
|
||||
Options:
|
||||
--limit N Maximum number of records to export (default: 50)
|
||||
--days N Only include failures from last N days (default: 30)
|
||||
--error-type TYPE Filter by specific error type (e.g., not_found, auth_error)
|
||||
--output FILE Output file path (default: failed_logs_export.json)
|
||||
--include-success Also include some successful "no new content" for comparison
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from sqlalchemy import create_engine, desc, or_
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.models.download import Download, DownloadStatus
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def export_failed_logs(
|
||||
limit: int = 50,
|
||||
days: int = 30,
|
||||
error_type: str | None = None,
|
||||
include_success: bool = False,
|
||||
output_path: str = "failed_logs_export.json",
|
||||
):
|
||||
"""Export failed download logs to JSON for analysis."""
|
||||
|
||||
settings = get_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
try:
|
||||
cutoff_date = datetime.utcnow() - timedelta(days=days)
|
||||
|
||||
# Build query for failed downloads
|
||||
query = session.query(Download).filter(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.created_at >= cutoff_date,
|
||||
)
|
||||
|
||||
if error_type:
|
||||
query = query.filter(Download.error_type == error_type)
|
||||
|
||||
# Order by most recent first
|
||||
failed_downloads = query.order_by(desc(Download.created_at)).limit(limit).all()
|
||||
|
||||
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||
success_downloads = []
|
||||
if include_success:
|
||||
success_query = session.query(Download).filter(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.created_at >= cutoff_date,
|
||||
Download.file_count == 0, # No new content cases
|
||||
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||
success_downloads = success_query.all()
|
||||
|
||||
# Format for export
|
||||
exports = []
|
||||
|
||||
for download in failed_downloads:
|
||||
exports.append(format_download_for_export(download, "failed"))
|
||||
|
||||
for download in success_downloads:
|
||||
exports.append(format_download_for_export(download, "success_no_new_content"))
|
||||
|
||||
# Build output document
|
||||
output = {
|
||||
"export_info": {
|
||||
"exported_at": datetime.utcnow().isoformat(),
|
||||
"total_records": len(exports),
|
||||
"failed_count": len(failed_downloads),
|
||||
"success_comparison_count": len(success_downloads),
|
||||
"days_included": days,
|
||||
"error_type_filter": error_type,
|
||||
},
|
||||
"analysis_prompt": generate_analysis_prompt(),
|
||||
"downloads": exports,
|
||||
}
|
||||
|
||||
# Write to file
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, indent=2, default=str)
|
||||
|
||||
print(f"Exported {len(exports)} records to {output_path}")
|
||||
print(f" - Failed: {len(failed_downloads)}")
|
||||
if include_success:
|
||||
print(f" - Success (no new content): {len(success_downloads)}")
|
||||
|
||||
# Print summary by error type
|
||||
error_types = {}
|
||||
for download in failed_downloads:
|
||||
et = download.error_type or "unknown"
|
||||
error_types[et] = error_types.get(et, 0) + 1
|
||||
|
||||
print("\nError type breakdown:")
|
||||
for et, count in sorted(error_types.items(), key=lambda x: -x[1]):
|
||||
print(f" {et}: {count}")
|
||||
|
||||
return output_path
|
||||
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def format_download_for_export(download: Download, classification: str) -> dict:
|
||||
"""Format a download record for export."""
|
||||
|
||||
# Extract logs from metadata
|
||||
metadata = download.metadata_ or {}
|
||||
stdout = metadata.get("stdout") or ""
|
||||
stderr = metadata.get("stderr") or ""
|
||||
|
||||
# Truncate very long logs but keep enough for analysis
|
||||
max_log_length = 50000 # 50KB per log field
|
||||
if stdout and len(stdout) > max_log_length:
|
||||
stdout = stdout[:max_log_length] + f"\n\n[... truncated, total length: {len(stdout)} chars ...]"
|
||||
if stderr and len(stderr) > max_log_length:
|
||||
stderr = stderr[:max_log_length] + f"\n\n[... truncated, total length: {len(stderr)} chars ...]"
|
||||
|
||||
# Get platform from source if available
|
||||
platform = None
|
||||
if download.source:
|
||||
platform = download.source.platform
|
||||
|
||||
return {
|
||||
"id": download.id,
|
||||
"classification": classification,
|
||||
"assigned_error_type": download.error_type,
|
||||
"assigned_error_message": download.error_message,
|
||||
"platform": platform,
|
||||
"url": download.url,
|
||||
"status": download.status, # Already a string, not enum
|
||||
"file_count": download.file_count,
|
||||
"created_at": download.created_at.isoformat() if download.created_at else None,
|
||||
"duration_seconds": metadata.get("duration_seconds"),
|
||||
"logs": {
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
},
|
||||
"user_feedback": None, # User can fill this in: "correct", "false_positive", "false_negative", "wrong_type"
|
||||
"suggested_error_type": None, # User can suggest what it should be
|
||||
"notes": None, # User can add context
|
||||
}
|
||||
|
||||
|
||||
def generate_analysis_prompt() -> str:
|
||||
"""Generate a prompt for Claude to analyze the logs."""
|
||||
return """
|
||||
## Error Log Analysis Task
|
||||
|
||||
I'm providing you with download logs from a gallery-dl based downloader. Each record includes:
|
||||
- The error_type that was assigned by the current classification logic
|
||||
- The actual stdout/stderr logs from gallery-dl
|
||||
- Platform and URL information
|
||||
|
||||
Please analyze these logs and identify:
|
||||
|
||||
1. **False Positives**: Cases where an error was reported but the download actually succeeded
|
||||
- Look for: successful HTTP responses (200), "skipping" messages indicating archive deduplication worked
|
||||
|
||||
2. **False Negatives**: Cases classified as one error type that should be another
|
||||
- Example: classified as "not_found" but logs show authentication issues
|
||||
|
||||
3. **Pattern Improvements**: Suggest more specific patterns to detect each error type
|
||||
- Current patterns may be too broad (matching debug output) or too narrow (missing variations)
|
||||
|
||||
4. **New Error Types**: Any failure modes not currently categorized
|
||||
|
||||
For each issue found, provide:
|
||||
- The download ID
|
||||
- Current classification vs suggested classification
|
||||
- The specific log lines that support your analysis
|
||||
- Suggested pattern improvements (if applicable)
|
||||
|
||||
Focus on actionable improvements to the _categorize_error() function in gallery_dl.py.
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Export failed download logs for error classification analysis"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=50,
|
||||
help="Maximum number of failed records to export (default: 50)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--days", type=int, default=30,
|
||||
help="Only include failures from last N days (default: 30)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--error-type", type=str, default=None,
|
||||
help="Filter by specific error type (e.g., not_found, auth_error)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", type=str, default="failed_logs_export.json",
|
||||
help="Output file path (default: failed_logs_export.json)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-success", action="store_true",
|
||||
help="Also include some successful 'no new content' records for comparison"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
export_failed_logs(
|
||||
limit=args.limit,
|
||||
days=args.days,
|
||||
error_type=args.error_type,
|
||||
include_success=args.include_success,
|
||||
output_path=args.output,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -55,6 +55,14 @@
|
||||
<v-list-item-title>Re-queue Stale Queued Jobs</v-list-item-title>
|
||||
<v-list-item-subtitle>Re-send lost Celery tasks for queued jobs</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider />
|
||||
<v-list-item @click="exportFailedLogs">
|
||||
<v-list-item-title>
|
||||
<v-icon start size="small">mdi-download</v-icon>
|
||||
Export Failed Logs
|
||||
</v-list-item-title>
|
||||
<v-list-item-subtitle>Download JSON for error analysis</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</v-col>
|
||||
@@ -382,6 +390,18 @@ async function requeueStaleJobs() {
|
||||
}
|
||||
}
|
||||
|
||||
async function exportFailedLogs() {
|
||||
try {
|
||||
// Open download in new tab - the API returns a file attachment
|
||||
const baseUrl = import.meta.env.VITE_API_URL || '/api'
|
||||
const url = `${baseUrl}/downloads/export-failed-logs?limit=100&days=30&include_success=true`
|
||||
window.open(url, '_blank')
|
||||
notifications.success('Exporting failed logs...')
|
||||
} catch (error) {
|
||||
notifications.error(`Failed to export logs: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
function getStatusIcon(status) {
|
||||
const icons = {
|
||||
completed: 'mdi-check-circle',
|
||||
|
||||
+42
-1
@@ -1,6 +1,6 @@
|
||||
# GallerySubscriber - Project Summary
|
||||
|
||||
**Updated:** 2026-01-29 (Redis DB config, mismatch detection, stale job recovery, Downloads UI improvements)
|
||||
**Updated:** 2026-01-30 (Error recognition improvements, analysis tooling)
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -368,3 +368,44 @@ Multiple layers prevent duplicate downloads for the same source:
|
||||
|
||||
3. **Scheduler Logic** (`downloads.py`):
|
||||
- `scheduled_check` skips sources with existing QUEUED or RUNNING downloads
|
||||
|
||||
## Error Classification System
|
||||
|
||||
The `_categorize_error()` method in `gallery_dl.py` classifies download failures into actionable error types:
|
||||
|
||||
| Error Type | Meaning | User Action |
|
||||
|------------|---------|-------------|
|
||||
| `no_new_content` | All content already downloaded (success) | None needed |
|
||||
| `auth_error` | Cookies expired or invalid | Re-export cookies from extension |
|
||||
| `rate_limited` | Too many requests (429) | Increase rate_limit setting |
|
||||
| `not_found` | Creator/content deleted or moved | Check URL, may need to remove |
|
||||
| `access_denied` | Insufficient subscription tier | Upgrade pledge or remove source |
|
||||
| `network_error` | Connection issues | Check network, will auto-retry |
|
||||
| `timeout` | Download took too long | Will auto-retry |
|
||||
| `unsupported_url` | URL not recognized by gallery-dl | Check URL format |
|
||||
| `unknown_error` | Unclassified failure | Check logs for details |
|
||||
|
||||
**Classification Logic:**
|
||||
|
||||
1. **Skip Detection First** - Checks for `#` prefixed lines in stdout (gallery-dl's skip message format) and text patterns like "skipping", "already exists". If skips are detected with no actual errors, returns `no_new_content`.
|
||||
|
||||
2. **Actual Error Detection** - Looks for `][error]` log level, exceptions, and tracebacks before proceeding to pattern matching.
|
||||
|
||||
3. **Specific Patterns** - Uses targeted patterns (e.g., `"timed out"` not just `"timeout"`) to avoid false positives from config values in debug output.
|
||||
|
||||
## Backend Scripts
|
||||
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `backend/scripts/export_failed_logs.py` | Export failed download logs to JSON for analysis |
|
||||
|
||||
## Analysis Directory
|
||||
|
||||
The `analysis/` directory (gitignored) contains debugging and analysis artifacts:
|
||||
|
||||
- `failed_logs_export_*.json` - Exported download logs for analysis
|
||||
- `analysis_results.json` - Parsed analysis results
|
||||
- `ERROR_RECOGNITION_ANALYSIS.md` - Detailed findings report
|
||||
- Analysis scripts for pattern debugging
|
||||
|
||||
These files are excluded from version control but preserved locally for ongoing analysis.
|
||||
|
||||
Reference in New Issue
Block a user