Files
FabledScribe/src/scribe/services/projects.py
T
bvandeusenandClaude Opus 5 be3a0ffaf9
CI & Build / Python lint (push) Successful in 4s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 43s
CI & Build / integration (push) Successful in 2m33s
CI & Build / Python tests (push) Successful in 3m1s
CI & Build / Build & push image (push) Successful in 44s
fix(projects): batch the summary queries — the fan-out was exhausting the pool
Reported live: Projects and Snippets showed skeletons that never resolved,
/knowledge worked intermittently. The logs named it exactly:

    QueuePool limit of size 5 overflow 10 reached, connection timed out, 30.00
    GET /api/settings  500  30584.0ms
    GET /api/projects  200  30882.9ms

/api/projects was not hanging — it was waiting out the 30-second checkout
timeout and then returning 200 with summaries silently missing, because
_attach swallowed the TimeoutError. Nobody waits 31 seconds, so it read as a
hang.

THE SHAPE: routes/projects.py ran asyncio.gather over every project. Each
_attach called get_project_summary, which opened its own session for three
queries and then called get_project_milestone_summary — which opened one more
session PER MILESTONE. So 25 projects asked for roughly 250 concurrent
checkouts against a pool of 15 (SQLAlchemy's default 5 + 10 overflow).

That is why unrelated routes failed too. Snippets and /knowledge were never
broken; they queued behind the burst and inherited its timeout. /api/settings
returning 500 while /api/projects returned 200 is the same cause wearing two
faces.

The comment above the gather said "one backend pass instead of N+1 frontend
calls". It did remove the N+1 from the network — and recreated it against the
connection pool, where it is worse, because the browser had at least been
serialising those calls.

Now: get_project_summaries() does all projects in four queries and one session,
and get_project_milestone_summaries() does all milestones in two. Two sessions
total for the whole page, independent of how many projects exist.

The progress calculation is extracted to _progress_from_counts and shared by
both the batch and single paths, so the cancelled-exclusion rule cannot drift
into two versions that disagree about whether a milestone is finished.

Tests assert the SESSION COUNT, not just the values. An implementation that
returned identical output while opening a session per project would pass a
correctness test and reproduce the outage.

Deliberately NOT done: raising pool_size. It would move the cliff rather than
remove it, and this endpoint now needs two connections regardless of scale.

Closes #2384.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaYUaouG9jjhATyuxCKrQs
2026-08-02 19:09:36 -04:00

334 lines
12 KiB
Python

"""Project management service."""
import logging
from datetime import datetime, timezone
from sqlalchemy import func, select
from scribe.models import async_session
from scribe.models.note import Note
from scribe.models.project import Project, ProjectStatus
logger = logging.getLogger(__name__)
def _validate_status(status: str) -> str:
"""Coerce/validate a project status against ProjectStatus.
Canonical gate so the MCP create/update_project path (which passes status
straight through) can't persist an out-of-enum value — there's no DB CHECK.
"""
try:
return ProjectStatus(status).value
except ValueError:
raise ValueError(
f"Invalid status: {status!r}. Must be one of: {[s.value for s in ProjectStatus]}"
)
async def create_project(
user_id: int,
title: str,
description: str = "",
goal: str = "",
color: str | None = None,
status: str = "active",
) -> Project:
status = _validate_status(status)
async with async_session() as session:
project = Project(
user_id=user_id,
title=title,
description=description,
goal=goal,
color=color,
status=status,
)
session.add(project)
await session.commit()
await session.refresh(project)
return project
async def get_project(user_id: int, project_id: int) -> Project | None:
async with async_session() as session:
result = await session.execute(
select(Project).where(
Project.id == project_id, Project.user_id == user_id,
Project.deleted_at.is_(None),
)
)
return result.scalars().first()
async def get_project_by_title(user_id: int, title: str) -> Project | None:
async with async_session() as session:
result = await session.execute(
select(Project).where(
Project.user_id == user_id,
func.lower(Project.title) == func.lower(title.strip()),
Project.deleted_at.is_(None),
).limit(1)
)
return result.scalars().first()
async def get_or_create_project(user_id: int, title: str) -> Project:
project = await get_project_by_title(user_id, title)
if project:
return project
return await create_project(user_id, title=title)
async def list_projects(user_id: int, status: str | None = None) -> list[Project]:
async with async_session() as session:
query = select(Project).where(
Project.user_id == user_id, Project.deleted_at.is_(None)
)
if status:
query = query.where(Project.status == status)
query = query.order_by(Project.updated_at.desc())
result = await session.execute(query)
return list(result.scalars().all())
async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
async with async_session() as session:
result = await session.execute(
select(Project).where(
Project.id == project_id, Project.user_id == user_id,
Project.deleted_at.is_(None),
)
)
project = result.scalars().first()
if project is None:
return None
if "status" in fields and fields["status"] is not None:
fields["status"] = _validate_status(fields["status"])
for key, value in fields.items():
if hasattr(project, key):
setattr(project, key, value)
project.updated_at = datetime.now(timezone.utc)
await session.commit()
await session.refresh(project)
return project
async def delete_project(user_id: int, project_id: int) -> bool:
async with async_session() as session:
result = await session.execute(
select(Project).where(Project.id == project_id, Project.user_id == user_id)
)
project = result.scalars().first()
if project is None:
return False
# Unlink notes (cascade handled by DB ON DELETE SET NULL, but we delete project here)
await session.delete(project)
await session.commit()
return True
async def get_project_summaries(
user_id: int, project_ids: list[int]
) -> dict[int, dict]:
"""Summaries for MANY projects — four queries and one session, total.
Replaces an `asyncio.gather` over the per-project version below, which was
a nested fan-out: each project opened its own session for three queries,
then called the milestone summary, which opened one more per milestone. For
25 projects that asked for roughly 250 pooled connections at once against a
pool of 15 (SQLAlchemy's default 5 + 10 overflow), so most of them sat out
the 30-second checkout timeout and everything else on the instance queued
behind them — including unrelated routes, which is why /api/settings
returned 500 while /api/projects took 30.9s (#2384).
The comment it replaced said "one backend pass instead of N+1 frontend
calls". It did remove the N+1 from the network — and recreated it against
the connection pool, where it is worse: the browser had at least been
serialising those calls.
"""
if not project_ids:
return {}
async with async_session() as session:
task_rows = await session.execute(
select(Note.project_id, Note.status, func.count(Note.id))
.where(
Note.user_id == user_id,
Note.project_id.in_(project_ids),
Note.status.isnot(None),
Note.deleted_at.is_(None),
)
.group_by(Note.project_id, Note.status)
)
task_counts: dict[int, dict[str, int]] = {}
for project_id, status, count in task_rows.fetchall():
task_counts.setdefault(project_id, {})[status] = count
note_rows = await session.execute(
select(Note.project_id, func.count(Note.id))
.where(
Note.user_id == user_id,
Note.project_id.in_(project_ids),
Note.status.is_(None),
Note.deleted_at.is_(None),
)
.group_by(Note.project_id)
)
note_counts = {pid: count for pid, count in note_rows.fetchall()}
# Deliberately NOT filtered by deleted_at, matching the per-project
# version: "last activity" includes trashing something.
activity_rows = await session.execute(
select(Note.project_id, func.max(Note.updated_at))
.where(Note.user_id == user_id, Note.project_id.in_(project_ids))
.group_by(Note.project_id)
)
last_activity = {pid: ts for pid, ts in activity_rows.fetchall()}
from scribe.services.milestones import get_project_milestone_summaries
milestones = await get_project_milestone_summaries(user_id, project_ids)
return {
pid: {
# All three lifecycle keys present so consumers can sum without
# `?? 0` guards — the frontend declares them required, and
# `undefined + N` renders as NaN.
"task_counts": {
"todo": 0, "in_progress": 0, "done": 0,
**task_counts.get(pid, {}),
},
"note_count": note_counts.get(pid, 0),
"last_activity": (
last_activity[pid].isoformat() if last_activity.get(pid) else None
),
"milestone_summary": milestones.get(pid, []),
}
for pid in project_ids
}
async def get_project_summary(user_id: int, project_id: int) -> dict:
"""Return task counts by status, note count, and last activity."""
async with async_session() as session:
# Task counts by status
task_rows = await session.execute(
select(Note.status, func.count(Note.id))
.where(
Note.user_id == user_id,
Note.project_id == project_id,
Note.status.isnot(None),
Note.deleted_at.is_(None),
)
.group_by(Note.status)
)
# Initialise all three lifecycle keys to 0 so consumers can sum them
# safely without `?? 0` guards. Frontend interface declares all three
# as required; rendering `undefined + N` yields NaN.
task_counts: dict[str, int] = {"todo": 0, "in_progress": 0, "done": 0}
for status, count in task_rows.fetchall():
task_counts[status] = count
# Note count (non-tasks)
note_count_result = await session.scalar(
select(func.count(Note.id)).where(
Note.user_id == user_id,
Note.project_id == project_id,
Note.status.is_(None),
Note.deleted_at.is_(None),
)
)
note_count = note_count_result or 0
# Last activity
last_activity_result = await session.scalar(
select(func.max(Note.updated_at)).where(
Note.user_id == user_id,
Note.project_id == project_id,
)
)
from scribe.services.milestones import get_project_milestone_summary
milestone_summary = await get_project_milestone_summary(user_id, project_id)
return {
"task_counts": task_counts,
"note_count": note_count,
"last_activity": last_activity_result.isoformat() if last_activity_result else None,
"milestone_summary": milestone_summary,
}
# ---------------------------------------------------------------------------
# Shared-access variants (honour ProjectShare in addition to ownership)
# ---------------------------------------------------------------------------
async def get_project_for_user(
accessing_user_id: int, project_id: int
) -> tuple[Project, str] | None:
"""Returns (project, permission) if user has any access, else None."""
from scribe.services.access import get_project_permission
perm = await get_project_permission(accessing_user_id, project_id)
if perm is None:
return None
async with async_session() as session:
project = await session.get(Project, project_id)
return (project, perm) if project else None
async def list_projects_for_user(user_id: int, status: str | None = None) -> list[dict]:
"""Owned projects + shared projects, each dict has 'permission' field."""
from scribe.models.group import GroupMembership
from scribe.models.share import ProjectShare
from scribe.services.access import PERMISSION_RANK
owned = await list_projects(user_id, status)
owned_ids = {p.id for p in owned}
result = []
for p in owned:
d = p.to_dict() if hasattr(p, "to_dict") else {"id": p.id, "title": p.title}
d["permission"] = "owner"
result.append(d)
async with async_session() as session:
user_group_ids = (await session.execute(
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
)).scalars().all()
shared_direct = (await session.execute(
select(ProjectShare).where(ProjectShare.shared_with_user_id == user_id)
)).scalars().all()
shared_group: list[ProjectShare] = []
if user_group_ids:
shared_group = (await session.execute(
select(ProjectShare).where(
ProjectShare.shared_with_group_id.in_(user_group_ids)
)
)).scalars().all()
seen: dict[int, str] = {}
for share in list(shared_direct) + list(shared_group):
if share.project_id in owned_ids:
continue
prev = seen.get(share.project_id)
if prev is None or PERMISSION_RANK[share.permission] > PERMISSION_RANK[prev]:
seen[share.project_id] = share.permission
for pid, perm in seen.items():
if status:
async with async_session() as session:
proj = await session.get(Project, pid)
if not proj or proj.status != status:
continue
else:
async with async_session() as session:
proj = await session.get(Project, pid)
if proj:
d = proj.to_dict() if hasattr(proj, "to_dict") else {"id": proj.id, "title": proj.title}
d["permission"] = perm
d["is_shared"] = True
result.append(d)
return result