CI & Build / Python lint (push) Successful in 4s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 43s
CI & Build / integration (push) Successful in 2m33s
CI & Build / Python tests (push) Successful in 3m1s
CI & Build / Build & push image (push) Successful in 44s
Reported live: Projects and Snippets showed skeletons that never resolved,
/knowledge worked intermittently. The logs named it exactly:
QueuePool limit of size 5 overflow 10 reached, connection timed out, 30.00
GET /api/settings 500 30584.0ms
GET /api/projects 200 30882.9ms
/api/projects was not hanging — it was waiting out the 30-second checkout
timeout and then returning 200 with summaries silently missing, because
_attach swallowed the TimeoutError. Nobody waits 31 seconds, so it read as a
hang.
THE SHAPE: routes/projects.py ran asyncio.gather over every project. Each
_attach called get_project_summary, which opened its own session for three
queries and then called get_project_milestone_summary — which opened one more
session PER MILESTONE. So 25 projects asked for roughly 250 concurrent
checkouts against a pool of 15 (SQLAlchemy's default 5 + 10 overflow).
That is why unrelated routes failed too. Snippets and /knowledge were never
broken; they queued behind the burst and inherited its timeout. /api/settings
returning 500 while /api/projects returned 200 is the same cause wearing two
faces.
The comment above the gather said "one backend pass instead of N+1 frontend
calls". It did remove the N+1 from the network — and recreated it against the
connection pool, where it is worse, because the browser had at least been
serialising those calls.
Now: get_project_summaries() does all projects in four queries and one session,
and get_project_milestone_summaries() does all milestones in two. Two sessions
total for the whole page, independent of how many projects exist.
The progress calculation is extracted to _progress_from_counts and shared by
both the batch and single paths, so the cancelled-exclusion rule cannot drift
into two versions that disagree about whether a milestone is finished.
Tests assert the SESSION COUNT, not just the values. An implementation that
returned identical output while opening a session per project would pass a
correctness test and reproduce the outage.
Deliberately NOT done: raising pool_size. It would move the cliff rather than
remove it, and this endpoint now needs two connections regardless of scale.
Closes #2384.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs
334 lines
12 KiB
Python
334 lines
12 KiB
Python
"""Project management service."""
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from scribe.models import async_session
|
|
from scribe.models.note import Note
|
|
from scribe.models.project import Project, ProjectStatus
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _validate_status(status: str) -> str:
|
|
"""Coerce/validate a project status against ProjectStatus.
|
|
|
|
Canonical gate so the MCP create/update_project path (which passes status
|
|
straight through) can't persist an out-of-enum value — there's no DB CHECK.
|
|
"""
|
|
try:
|
|
return ProjectStatus(status).value
|
|
except ValueError:
|
|
raise ValueError(
|
|
f"Invalid status: {status!r}. Must be one of: {[s.value for s in ProjectStatus]}"
|
|
)
|
|
|
|
|
|
async def create_project(
|
|
user_id: int,
|
|
title: str,
|
|
description: str = "",
|
|
goal: str = "",
|
|
color: str | None = None,
|
|
status: str = "active",
|
|
) -> Project:
|
|
status = _validate_status(status)
|
|
async with async_session() as session:
|
|
project = Project(
|
|
user_id=user_id,
|
|
title=title,
|
|
description=description,
|
|
goal=goal,
|
|
color=color,
|
|
status=status,
|
|
)
|
|
session.add(project)
|
|
await session.commit()
|
|
await session.refresh(project)
|
|
return project
|
|
|
|
|
|
async def get_project(user_id: int, project_id: int) -> Project | None:
|
|
async with async_session() as session:
|
|
result = await session.execute(
|
|
select(Project).where(
|
|
Project.id == project_id, Project.user_id == user_id,
|
|
Project.deleted_at.is_(None),
|
|
)
|
|
)
|
|
return result.scalars().first()
|
|
|
|
|
|
async def get_project_by_title(user_id: int, title: str) -> Project | None:
|
|
async with async_session() as session:
|
|
result = await session.execute(
|
|
select(Project).where(
|
|
Project.user_id == user_id,
|
|
func.lower(Project.title) == func.lower(title.strip()),
|
|
Project.deleted_at.is_(None),
|
|
).limit(1)
|
|
)
|
|
return result.scalars().first()
|
|
|
|
|
|
async def get_or_create_project(user_id: int, title: str) -> Project:
|
|
project = await get_project_by_title(user_id, title)
|
|
if project:
|
|
return project
|
|
return await create_project(user_id, title=title)
|
|
|
|
|
|
async def list_projects(user_id: int, status: str | None = None) -> list[Project]:
|
|
async with async_session() as session:
|
|
query = select(Project).where(
|
|
Project.user_id == user_id, Project.deleted_at.is_(None)
|
|
)
|
|
if status:
|
|
query = query.where(Project.status == status)
|
|
query = query.order_by(Project.updated_at.desc())
|
|
result = await session.execute(query)
|
|
return list(result.scalars().all())
|
|
|
|
|
|
async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
|
|
async with async_session() as session:
|
|
result = await session.execute(
|
|
select(Project).where(
|
|
Project.id == project_id, Project.user_id == user_id,
|
|
Project.deleted_at.is_(None),
|
|
)
|
|
)
|
|
project = result.scalars().first()
|
|
if project is None:
|
|
return None
|
|
if "status" in fields and fields["status"] is not None:
|
|
fields["status"] = _validate_status(fields["status"])
|
|
for key, value in fields.items():
|
|
if hasattr(project, key):
|
|
setattr(project, key, value)
|
|
project.updated_at = datetime.now(timezone.utc)
|
|
await session.commit()
|
|
await session.refresh(project)
|
|
return project
|
|
|
|
|
|
async def delete_project(user_id: int, project_id: int) -> bool:
|
|
async with async_session() as session:
|
|
result = await session.execute(
|
|
select(Project).where(Project.id == project_id, Project.user_id == user_id)
|
|
)
|
|
project = result.scalars().first()
|
|
if project is None:
|
|
return False
|
|
# Unlink notes (cascade handled by DB ON DELETE SET NULL, but we delete project here)
|
|
await session.delete(project)
|
|
await session.commit()
|
|
return True
|
|
|
|
|
|
async def get_project_summaries(
|
|
user_id: int, project_ids: list[int]
|
|
) -> dict[int, dict]:
|
|
"""Summaries for MANY projects — four queries and one session, total.
|
|
|
|
Replaces an `asyncio.gather` over the per-project version below, which was
|
|
a nested fan-out: each project opened its own session for three queries,
|
|
then called the milestone summary, which opened one more per milestone. For
|
|
25 projects that asked for roughly 250 pooled connections at once against a
|
|
pool of 15 (SQLAlchemy's default 5 + 10 overflow), so most of them sat out
|
|
the 30-second checkout timeout and everything else on the instance queued
|
|
behind them — including unrelated routes, which is why /api/settings
|
|
returned 500 while /api/projects took 30.9s (#2384).
|
|
|
|
The comment it replaced said "one backend pass instead of N+1 frontend
|
|
calls". It did remove the N+1 from the network — and recreated it against
|
|
the connection pool, where it is worse: the browser had at least been
|
|
serialising those calls.
|
|
"""
|
|
if not project_ids:
|
|
return {}
|
|
|
|
async with async_session() as session:
|
|
task_rows = await session.execute(
|
|
select(Note.project_id, Note.status, func.count(Note.id))
|
|
.where(
|
|
Note.user_id == user_id,
|
|
Note.project_id.in_(project_ids),
|
|
Note.status.isnot(None),
|
|
Note.deleted_at.is_(None),
|
|
)
|
|
.group_by(Note.project_id, Note.status)
|
|
)
|
|
task_counts: dict[int, dict[str, int]] = {}
|
|
for project_id, status, count in task_rows.fetchall():
|
|
task_counts.setdefault(project_id, {})[status] = count
|
|
|
|
note_rows = await session.execute(
|
|
select(Note.project_id, func.count(Note.id))
|
|
.where(
|
|
Note.user_id == user_id,
|
|
Note.project_id.in_(project_ids),
|
|
Note.status.is_(None),
|
|
Note.deleted_at.is_(None),
|
|
)
|
|
.group_by(Note.project_id)
|
|
)
|
|
note_counts = {pid: count for pid, count in note_rows.fetchall()}
|
|
|
|
# Deliberately NOT filtered by deleted_at, matching the per-project
|
|
# version: "last activity" includes trashing something.
|
|
activity_rows = await session.execute(
|
|
select(Note.project_id, func.max(Note.updated_at))
|
|
.where(Note.user_id == user_id, Note.project_id.in_(project_ids))
|
|
.group_by(Note.project_id)
|
|
)
|
|
last_activity = {pid: ts for pid, ts in activity_rows.fetchall()}
|
|
|
|
from scribe.services.milestones import get_project_milestone_summaries
|
|
milestones = await get_project_milestone_summaries(user_id, project_ids)
|
|
|
|
return {
|
|
pid: {
|
|
# All three lifecycle keys present so consumers can sum without
|
|
# `?? 0` guards — the frontend declares them required, and
|
|
# `undefined + N` renders as NaN.
|
|
"task_counts": {
|
|
"todo": 0, "in_progress": 0, "done": 0,
|
|
**task_counts.get(pid, {}),
|
|
},
|
|
"note_count": note_counts.get(pid, 0),
|
|
"last_activity": (
|
|
last_activity[pid].isoformat() if last_activity.get(pid) else None
|
|
),
|
|
"milestone_summary": milestones.get(pid, []),
|
|
}
|
|
for pid in project_ids
|
|
}
|
|
|
|
|
|
async def get_project_summary(user_id: int, project_id: int) -> dict:
|
|
"""Return task counts by status, note count, and last activity."""
|
|
async with async_session() as session:
|
|
# Task counts by status
|
|
task_rows = await session.execute(
|
|
select(Note.status, func.count(Note.id))
|
|
.where(
|
|
Note.user_id == user_id,
|
|
Note.project_id == project_id,
|
|
Note.status.isnot(None),
|
|
Note.deleted_at.is_(None),
|
|
)
|
|
.group_by(Note.status)
|
|
)
|
|
# Initialise all three lifecycle keys to 0 so consumers can sum them
|
|
# safely without `?? 0` guards. Frontend interface declares all three
|
|
# as required; rendering `undefined + N` yields NaN.
|
|
task_counts: dict[str, int] = {"todo": 0, "in_progress": 0, "done": 0}
|
|
for status, count in task_rows.fetchall():
|
|
task_counts[status] = count
|
|
|
|
# Note count (non-tasks)
|
|
note_count_result = await session.scalar(
|
|
select(func.count(Note.id)).where(
|
|
Note.user_id == user_id,
|
|
Note.project_id == project_id,
|
|
Note.status.is_(None),
|
|
Note.deleted_at.is_(None),
|
|
)
|
|
)
|
|
note_count = note_count_result or 0
|
|
|
|
# Last activity
|
|
last_activity_result = await session.scalar(
|
|
select(func.max(Note.updated_at)).where(
|
|
Note.user_id == user_id,
|
|
Note.project_id == project_id,
|
|
)
|
|
)
|
|
|
|
from scribe.services.milestones import get_project_milestone_summary
|
|
milestone_summary = await get_project_milestone_summary(user_id, project_id)
|
|
|
|
return {
|
|
"task_counts": task_counts,
|
|
"note_count": note_count,
|
|
"last_activity": last_activity_result.isoformat() if last_activity_result else None,
|
|
"milestone_summary": milestone_summary,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared-access variants (honour ProjectShare in addition to ownership)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_project_for_user(
|
|
accessing_user_id: int, project_id: int
|
|
) -> tuple[Project, str] | None:
|
|
"""Returns (project, permission) if user has any access, else None."""
|
|
from scribe.services.access import get_project_permission
|
|
perm = await get_project_permission(accessing_user_id, project_id)
|
|
if perm is None:
|
|
return None
|
|
async with async_session() as session:
|
|
project = await session.get(Project, project_id)
|
|
return (project, perm) if project else None
|
|
|
|
|
|
async def list_projects_for_user(user_id: int, status: str | None = None) -> list[dict]:
|
|
"""Owned projects + shared projects, each dict has 'permission' field."""
|
|
from scribe.models.group import GroupMembership
|
|
from scribe.models.share import ProjectShare
|
|
from scribe.services.access import PERMISSION_RANK
|
|
|
|
owned = await list_projects(user_id, status)
|
|
owned_ids = {p.id for p in owned}
|
|
|
|
result = []
|
|
for p in owned:
|
|
d = p.to_dict() if hasattr(p, "to_dict") else {"id": p.id, "title": p.title}
|
|
d["permission"] = "owner"
|
|
result.append(d)
|
|
|
|
async with async_session() as session:
|
|
user_group_ids = (await session.execute(
|
|
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
|
)).scalars().all()
|
|
|
|
shared_direct = (await session.execute(
|
|
select(ProjectShare).where(ProjectShare.shared_with_user_id == user_id)
|
|
)).scalars().all()
|
|
|
|
shared_group: list[ProjectShare] = []
|
|
if user_group_ids:
|
|
shared_group = (await session.execute(
|
|
select(ProjectShare).where(
|
|
ProjectShare.shared_with_group_id.in_(user_group_ids)
|
|
)
|
|
)).scalars().all()
|
|
|
|
seen: dict[int, str] = {}
|
|
for share in list(shared_direct) + list(shared_group):
|
|
if share.project_id in owned_ids:
|
|
continue
|
|
prev = seen.get(share.project_id)
|
|
if prev is None or PERMISSION_RANK[share.permission] > PERMISSION_RANK[prev]:
|
|
seen[share.project_id] = share.permission
|
|
|
|
for pid, perm in seen.items():
|
|
if status:
|
|
async with async_session() as session:
|
|
proj = await session.get(Project, pid)
|
|
if not proj or proj.status != status:
|
|
continue
|
|
else:
|
|
async with async_session() as session:
|
|
proj = await session.get(Project, pid)
|
|
if proj:
|
|
d = proj.to_dict() if hasattr(proj, "to_dict") else {"id": proj.id, "title": proj.title}
|
|
d["permission"] = perm
|
|
d["is_shared"] = True
|
|
result.append(d)
|
|
|
|
return result
|