fix: wrap AsyncSession in context manager across all API blueprints
Replace all bare `session = AsyncSession(bind=conn)` assignments with `async with AsyncSession(bind=conn) as session:` in downloads.py, credentials.py, settings.py, sources.py, and subscriptions.py to ensure sessions are properly closed on all code paths. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+218
-224
@@ -28,124 +28,121 @@ async def list_downloads():
|
||||
per_page = min(int(request.args.get("per_page", 50)), 100)
|
||||
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
# Build query - eager load source and subscription for labeling
|
||||
query = select(Download).options(
|
||||
selectinload(Download.source).selectinload(Source.subscription)
|
||||
)
|
||||
count_query = select(func.count()).select_from(Download)
|
||||
|
||||
# Build query - eager load source and subscription for labeling
|
||||
query = select(Download).options(
|
||||
selectinload(Download.source).selectinload(Source.subscription)
|
||||
)
|
||||
count_query = select(func.count()).select_from(Download)
|
||||
filters = []
|
||||
if source_id:
|
||||
filters.append(Download.source_id == source_id)
|
||||
if status:
|
||||
filters.append(Download.status == status)
|
||||
if from_date:
|
||||
filters.append(Download.created_at >= datetime.fromisoformat(from_date))
|
||||
if to_date:
|
||||
filters.append(Download.created_at <= datetime.fromisoformat(to_date))
|
||||
# When filtering failed downloads, optionally exclude ones superseded by a later success
|
||||
if exclude_superseded and status == DownloadStatus.FAILED:
|
||||
filters.append(Download.superseded == False)
|
||||
|
||||
filters = []
|
||||
if source_id:
|
||||
filters.append(Download.source_id == source_id)
|
||||
if status:
|
||||
filters.append(Download.status == status)
|
||||
if from_date:
|
||||
filters.append(Download.created_at >= datetime.fromisoformat(from_date))
|
||||
if to_date:
|
||||
filters.append(Download.created_at <= datetime.fromisoformat(to_date))
|
||||
# When filtering failed downloads, optionally exclude ones superseded by a later success
|
||||
if exclude_superseded and status == DownloadStatus.FAILED:
|
||||
filters.append(Download.superseded == False)
|
||||
if filters:
|
||||
query = query.where(and_(*filters))
|
||||
count_query = count_query.where(and_(*filters))
|
||||
|
||||
if filters:
|
||||
query = query.where(and_(*filters))
|
||||
count_query = count_query.where(and_(*filters))
|
||||
# Get total count
|
||||
total_result = await session.execute(count_query)
|
||||
total = total_result.scalar()
|
||||
|
||||
# Get total count
|
||||
total_result = await session.execute(count_query)
|
||||
total = total_result.scalar()
|
||||
# Apply pagination and ordering
|
||||
query = query.order_by(Download.created_at.desc())
|
||||
query = query.offset((page - 1) * per_page).limit(per_page)
|
||||
|
||||
# Apply pagination and ordering
|
||||
query = query.order_by(Download.created_at.desc())
|
||||
query = query.offset((page - 1) * per_page).limit(per_page)
|
||||
result = await session.execute(query)
|
||||
downloads = result.scalars().all()
|
||||
|
||||
result = await session.execute(query)
|
||||
downloads = result.scalars().all()
|
||||
# Include source info with each download
|
||||
items = []
|
||||
for d in downloads:
|
||||
item = d.to_dict()
|
||||
if d.source:
|
||||
item["subscription_name"] = d.source.subscription.name if d.source.subscription else None
|
||||
item["platform"] = d.source.platform
|
||||
else:
|
||||
item["subscription_name"] = None
|
||||
item["platform"] = None
|
||||
items.append(item)
|
||||
|
||||
# Include source info with each download
|
||||
items = []
|
||||
for d in downloads:
|
||||
item = d.to_dict()
|
||||
if d.source:
|
||||
item["subscription_name"] = d.source.subscription.name if d.source.subscription else None
|
||||
item["platform"] = d.source.platform
|
||||
else:
|
||||
item["subscription_name"] = None
|
||||
item["platform"] = None
|
||||
items.append(item)
|
||||
|
||||
return jsonify({
|
||||
"items": items,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page,
|
||||
})
|
||||
return jsonify({
|
||||
"items": items,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page,
|
||||
})
|
||||
|
||||
|
||||
@bp.route("/<int:download_id>", methods=["GET"])
|
||||
async def get_download(download_id: int):
|
||||
"""Get download details."""
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
result = await session.execute(select(Download).where(Download.id == download_id))
|
||||
download = result.scalar()
|
||||
|
||||
result = await session.execute(select(Download).where(Download.id == download_id))
|
||||
download = result.scalar()
|
||||
if not download:
|
||||
return jsonify({"error": "Download not found"}), 404
|
||||
|
||||
if not download:
|
||||
return jsonify({"error": "Download not found"}), 404
|
||||
# Include source info if available
|
||||
response = download.to_dict()
|
||||
if download.source_id:
|
||||
source_result = await session.execute(
|
||||
select(Source)
|
||||
.options(selectinload(Source.subscription))
|
||||
.where(Source.id == download.source_id)
|
||||
)
|
||||
source = source_result.scalar()
|
||||
if source:
|
||||
response["source"] = {
|
||||
"id": source.id,
|
||||
"subscription_name": source.subscription.name if source.subscription else None,
|
||||
"platform": source.platform,
|
||||
}
|
||||
|
||||
# Include source info if available
|
||||
response = download.to_dict()
|
||||
if download.source_id:
|
||||
source_result = await session.execute(
|
||||
select(Source)
|
||||
.options(selectinload(Source.subscription))
|
||||
.where(Source.id == download.source_id)
|
||||
)
|
||||
source = source_result.scalar()
|
||||
if source:
|
||||
response["source"] = {
|
||||
"id": source.id,
|
||||
"subscription_name": source.subscription.name if source.subscription else None,
|
||||
"platform": source.platform,
|
||||
}
|
||||
|
||||
return jsonify(response)
|
||||
return jsonify(response)
|
||||
|
||||
|
||||
@bp.route("/<int:download_id>/retry", methods=["POST"])
|
||||
async def retry_download(download_id: int):
|
||||
"""Retry a failed download."""
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
result = await session.execute(select(Download).where(Download.id == download_id))
|
||||
download = result.scalar()
|
||||
|
||||
result = await session.execute(select(Download).where(Download.id == download_id))
|
||||
download = result.scalar()
|
||||
if not download:
|
||||
return jsonify({"error": "Download not found"}), 404
|
||||
|
||||
if not download:
|
||||
return jsonify({"error": "Download not found"}), 404
|
||||
if download.status != DownloadStatus.FAILED:
|
||||
return jsonify({"error": "Can only retry failed downloads"}), 400
|
||||
|
||||
if download.status != DownloadStatus.FAILED:
|
||||
return jsonify({"error": "Can only retry failed downloads"}), 400
|
||||
# Reset status to queued
|
||||
download.status = DownloadStatus.QUEUED
|
||||
download.error_type = None
|
||||
download.error_message = None
|
||||
await session.commit()
|
||||
|
||||
# Reset status to queued
|
||||
download.status = DownloadStatus.QUEUED
|
||||
download.error_type = None
|
||||
download.error_message = None
|
||||
await session.commit()
|
||||
# Queue Celery task to retry the download
|
||||
task = process_download.delay(download_id)
|
||||
|
||||
# Queue Celery task to retry the download
|
||||
task = process_download.delay(download_id)
|
||||
|
||||
current_app.logger.info(f"Queued retry for download: {download_id}")
|
||||
return jsonify({
|
||||
"message": "Retry queued",
|
||||
"download_id": download_id,
|
||||
"task_id": task.id,
|
||||
}), 202
|
||||
current_app.logger.info(f"Queued retry for download: {download_id}")
|
||||
return jsonify({
|
||||
"message": "Retry queued",
|
||||
"download_id": download_id,
|
||||
"task_id": task.id,
|
||||
}), 202
|
||||
|
||||
|
||||
@bp.route("/recent-activity", methods=["GET"])
|
||||
@@ -162,48 +159,47 @@ async def recent_activity():
|
||||
cutoff = utcnow() - timedelta(days=days)
|
||||
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
|
||||
# Get failed downloads OR completed with files (noteworthy activity)
|
||||
# Exclude failed downloads that have been superseded by a later success
|
||||
# for the same source - those are stale failures the user doesn't need to see
|
||||
query = select(Download).options(
|
||||
selectinload(Download.source).selectinload(Source.subscription)
|
||||
).where(
|
||||
and_(
|
||||
Download.created_at >= cutoff,
|
||||
or_(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.superseded == False,
|
||||
),
|
||||
and_(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.file_count > 0
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
# Get failed downloads OR completed with files (noteworthy activity)
|
||||
# Exclude failed downloads that have been superseded by a later success
|
||||
# for the same source - those are stale failures the user doesn't need to see
|
||||
query = select(Download).options(
|
||||
selectinload(Download.source).selectinload(Source.subscription)
|
||||
).where(
|
||||
and_(
|
||||
Download.created_at >= cutoff,
|
||||
or_(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.superseded == False,
|
||||
),
|
||||
and_(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.file_count > 0
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
).order_by(Download.created_at.desc()).limit(limit)
|
||||
).order_by(Download.created_at.desc()).limit(limit)
|
||||
|
||||
result = await session.execute(query)
|
||||
downloads = result.scalars().all()
|
||||
result = await session.execute(query)
|
||||
downloads = result.scalars().all()
|
||||
|
||||
# Include source info with each download
|
||||
items = []
|
||||
for d in downloads:
|
||||
item = d.to_dict()
|
||||
if d.source:
|
||||
item["subscription_name"] = d.source.subscription.name if d.source.subscription else None
|
||||
item["platform"] = d.source.platform
|
||||
else:
|
||||
item["subscription_name"] = None
|
||||
item["platform"] = None
|
||||
items.append(item)
|
||||
# Include source info with each download
|
||||
items = []
|
||||
for d in downloads:
|
||||
item = d.to_dict()
|
||||
if d.source:
|
||||
item["subscription_name"] = d.source.subscription.name if d.source.subscription else None
|
||||
item["platform"] = d.source.platform
|
||||
else:
|
||||
item["subscription_name"] = None
|
||||
item["platform"] = None
|
||||
items.append(item)
|
||||
|
||||
return jsonify({
|
||||
"items": items,
|
||||
"count": len(downloads),
|
||||
})
|
||||
return jsonify({
|
||||
"items": items,
|
||||
"count": len(downloads),
|
||||
})
|
||||
|
||||
|
||||
@bp.route("/reset-orphaned", methods=["POST"])
|
||||
@@ -286,54 +282,53 @@ async def get_stats():
|
||||
period = request.args.get("period", "week") # day, week, month
|
||||
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
# Total counts by status
|
||||
status_query = select(
|
||||
Download.status,
|
||||
func.count(Download.id)
|
||||
).group_by(Download.status)
|
||||
|
||||
# Total counts by status
|
||||
status_query = select(
|
||||
Download.status,
|
||||
func.count(Download.id)
|
||||
).group_by(Download.status)
|
||||
status_result = await session.execute(status_query)
|
||||
status_counts = {row[0]: row[1] for row in status_result}
|
||||
|
||||
status_result = await session.execute(status_query)
|
||||
status_counts = {row[0]: row[1] for row in status_result}
|
||||
# Counts by platform (through source)
|
||||
platform_query = select(
|
||||
Source.platform,
|
||||
func.count(Download.id)
|
||||
).join(Source, Download.source_id == Source.id).group_by(Source.platform)
|
||||
|
||||
# Counts by platform (through source)
|
||||
platform_query = select(
|
||||
Source.platform,
|
||||
func.count(Download.id)
|
||||
).join(Source, Download.source_id == Source.id).group_by(Source.platform)
|
||||
platform_result = await session.execute(platform_query)
|
||||
platform_counts = {row[0]: row[1] for row in platform_result}
|
||||
|
||||
platform_result = await session.execute(platform_query)
|
||||
platform_counts = {row[0]: row[1] for row in platform_result}
|
||||
|
||||
# Recent downloads count
|
||||
recent_query = select(func.count()).select_from(Download).where(
|
||||
Download.status == DownloadStatus.COMPLETED
|
||||
)
|
||||
recent_result = await session.execute(recent_query)
|
||||
recent_completed = recent_result.scalar()
|
||||
|
||||
# Active failed: failures not superseded by a later success for the same source
|
||||
active_failed_query = select(func.count()).select_from(Download).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.superseded == False,
|
||||
# Recent downloads count
|
||||
recent_query = select(func.count()).select_from(Download).where(
|
||||
Download.status == DownloadStatus.COMPLETED
|
||||
)
|
||||
)
|
||||
active_failed_result = await session.execute(active_failed_query)
|
||||
active_failed = active_failed_result.scalar()
|
||||
recent_result = await session.execute(recent_query)
|
||||
recent_completed = recent_result.scalar()
|
||||
|
||||
return jsonify({
|
||||
"total": sum(status_counts.values()),
|
||||
"by_status": status_counts,
|
||||
"by_platform": platform_counts,
|
||||
"completed": status_counts.get(DownloadStatus.COMPLETED, 0),
|
||||
"failed": status_counts.get(DownloadStatus.FAILED, 0),
|
||||
"active_failed": active_failed,
|
||||
"pending": status_counts.get(DownloadStatus.PENDING, 0),
|
||||
"queued": status_counts.get(DownloadStatus.QUEUED, 0),
|
||||
"running": status_counts.get(DownloadStatus.RUNNING, 0),
|
||||
})
|
||||
# Active failed: failures not superseded by a later success for the same source
|
||||
active_failed_query = select(func.count()).select_from(Download).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.superseded == False,
|
||||
)
|
||||
)
|
||||
active_failed_result = await session.execute(active_failed_query)
|
||||
active_failed = active_failed_result.scalar()
|
||||
|
||||
return jsonify({
|
||||
"total": sum(status_counts.values()),
|
||||
"by_status": status_counts,
|
||||
"by_platform": platform_counts,
|
||||
"completed": status_counts.get(DownloadStatus.COMPLETED, 0),
|
||||
"failed": status_counts.get(DownloadStatus.FAILED, 0),
|
||||
"active_failed": active_failed,
|
||||
"pending": status_counts.get(DownloadStatus.PENDING, 0),
|
||||
"queued": status_counts.get(DownloadStatus.QUEUED, 0),
|
||||
"running": status_counts.get(DownloadStatus.RUNNING, 0),
|
||||
})
|
||||
|
||||
|
||||
@bp.route("/export-failed-logs", methods=["GET"])
|
||||
@@ -356,69 +351,68 @@ async def export_failed_logs():
|
||||
cutoff_date = utcnow() - timedelta(days=days)
|
||||
|
||||
async with current_app.db_engine.connect() as conn:
|
||||
session = AsyncSession(bind=conn)
|
||||
|
||||
# Build query for failed downloads (eager load source for platform)
|
||||
query = select(Download).options(selectinload(Download.source)).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.created_at >= cutoff_date,
|
||||
)
|
||||
)
|
||||
|
||||
if error_type:
|
||||
query = query.where(Download.error_type == error_type)
|
||||
|
||||
# Order by most recent first
|
||||
query = query.order_by(desc(Download.created_at)).limit(limit)
|
||||
result = await session.execute(query)
|
||||
failed_downloads = result.scalars().all()
|
||||
|
||||
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||
success_downloads = []
|
||||
if include_success:
|
||||
success_query = select(Download).options(selectinload(Download.source)).where(
|
||||
async with AsyncSession(bind=conn) as session:
|
||||
# Build query for failed downloads (eager load source for platform)
|
||||
query = select(Download).options(selectinload(Download.source)).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.status == DownloadStatus.FAILED,
|
||||
Download.created_at >= cutoff_date,
|
||||
Download.file_count == 0, # No new content cases
|
||||
)
|
||||
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||
success_result = await session.execute(success_query)
|
||||
success_downloads = success_result.scalars().all()
|
||||
)
|
||||
|
||||
# Format for export
|
||||
exports = []
|
||||
if error_type:
|
||||
query = query.where(Download.error_type == error_type)
|
||||
|
||||
for download in failed_downloads:
|
||||
exports.append(_format_download_for_export(download, "failed"))
|
||||
# Order by most recent first
|
||||
query = query.order_by(desc(Download.created_at)).limit(limit)
|
||||
result = await session.execute(query)
|
||||
failed_downloads = result.scalars().all()
|
||||
|
||||
for download in success_downloads:
|
||||
exports.append(_format_download_for_export(download, "success_no_new_content"))
|
||||
# Optionally include some successful NO_NEW_CONTENT for comparison
|
||||
success_downloads = []
|
||||
if include_success:
|
||||
success_query = select(Download).options(selectinload(Download.source)).where(
|
||||
and_(
|
||||
Download.status == DownloadStatus.COMPLETED,
|
||||
Download.created_at >= cutoff_date,
|
||||
Download.file_count == 0, # No new content cases
|
||||
)
|
||||
).order_by(desc(Download.created_at)).limit(limit // 4)
|
||||
success_result = await session.execute(success_query)
|
||||
success_downloads = success_result.scalars().all()
|
||||
|
||||
# Build output document
|
||||
output = {
|
||||
"export_info": {
|
||||
"exported_at": utcnow().isoformat(),
|
||||
"total_records": len(exports),
|
||||
"failed_count": len(failed_downloads),
|
||||
"success_comparison_count": len(success_downloads),
|
||||
"days_included": days,
|
||||
"error_type_filter": error_type,
|
||||
},
|
||||
"analysis_prompt": _generate_analysis_prompt(),
|
||||
"downloads": exports,
|
||||
}
|
||||
# Format for export
|
||||
exports = []
|
||||
|
||||
# Return as downloadable JSON file
|
||||
response = Response(
|
||||
json.dumps(output, indent=2, default=str),
|
||||
mimetype="application/json",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=failed_logs_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||||
for download in failed_downloads:
|
||||
exports.append(_format_download_for_export(download, "failed"))
|
||||
|
||||
for download in success_downloads:
|
||||
exports.append(_format_download_for_export(download, "success_no_new_content"))
|
||||
|
||||
# Build output document
|
||||
output = {
|
||||
"export_info": {
|
||||
"exported_at": utcnow().isoformat(),
|
||||
"total_records": len(exports),
|
||||
"failed_count": len(failed_downloads),
|
||||
"success_comparison_count": len(success_downloads),
|
||||
"days_included": days,
|
||||
"error_type_filter": error_type,
|
||||
},
|
||||
"analysis_prompt": _generate_analysis_prompt(),
|
||||
"downloads": exports,
|
||||
}
|
||||
)
|
||||
return response
|
||||
|
||||
# Return as downloadable JSON file
|
||||
response = Response(
|
||||
json.dumps(output, indent=2, default=str),
|
||||
mimetype="application/json",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=failed_logs_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||||
}
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _format_download_for_export(download: Download, classification: str) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user