rename(547): the server is grimoire — package, image, DB defaults, backups
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 56s
CI & Build / integration (push) Successful in 1m4s
CI & Build / Python tests (push) Successful in 1m55s
CI & Build / Build & push image (push) Successful in 44s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 12s
CI & Build / TypeScript typecheck (push) Successful in 56s
CI & Build / integration (push) Successful in 1m4s
CI & Build / Python tests (push) Successful in 1m55s
CI & Build / Build & push image (push) Successful in 44s
Step 2 of the Scribe -> Grimoire rename. - src/scribe -> src/grimoire, and every import, patch target, module path, hypercorn entry point and `python -m grimoire.restore` with it; pyproject, uv.lock and the frontend package are named grimoire. - Image git.fabledsword.com/bvandeusen/fabledgrimoire; CI's Postgres service runs as grimoire / grimoire_test. - Compose files default the DB user, database and (prod) volume to grimoire, each overridable from .env, so an install created as Scribe keeps its data by setting POSTGRES_USER, POSTGRES_DB and PGDATA_VOLUME. - Backups are written as grimoire-<ts>.tar with format grimoire-backup-tar. Archives named scribe-* with the scribe-backup-tar manifest still restore and are still counted at the destination (folder and S3), with tests. - Exports, the MCP auth realm, the ASGI scope key and the persistent logger carry the new name. Kept on purpose: the secret-sealing key label stays "scribe.secrets.v1", because every stored secret was derived under it. Plugin identity (plugin name, hooks, env vars, .scribe marker, skills) is step 3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,449 @@
|
||||
"""Access control service.
|
||||
|
||||
Single source of truth for permission resolution on projects and notes.
|
||||
All human-facing routes that should honour shares call these functions.
|
||||
LLM tool routes continue to use owner-scoped service functions directly.
|
||||
|
||||
Permission rank: owner > admin > editor > viewer
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.design_system import DesignSystem
|
||||
from grimoire.models.group import GroupMembership
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.share import NoteShare, ProjectShare
|
||||
from grimoire.models.user import User
|
||||
from grimoire.services.design_cascade import ancestry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PERMISSION_RANK: dict[str, int] = {
|
||||
"viewer": 1,
|
||||
"editor": 2,
|
||||
"admin": 3,
|
||||
"owner": 4,
|
||||
}
|
||||
|
||||
|
||||
def _higher(a: str | None, b: str | None) -> str | None:
|
||||
"""Return the higher-ranked permission, or None if both are None."""
|
||||
if a is None:
|
||||
return b
|
||||
if b is None:
|
||||
return a
|
||||
return a if PERMISSION_RANK[a] >= PERMISSION_RANK[b] else b
|
||||
|
||||
|
||||
async def _user_group_ids(session, user_id: int) -> list[int]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
||||
)
|
||||
).scalars().all()
|
||||
return list(rows)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project permissions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_project_permission(user_id: int, project_id: int) -> str | None:
|
||||
"""Return the effective permission string for user on project, or None."""
|
||||
async with async_session() as session:
|
||||
project = await session.get(Project, project_id)
|
||||
if project is None:
|
||||
return None
|
||||
if project.user_id == user_id:
|
||||
return "owner"
|
||||
|
||||
# Direct share
|
||||
direct = (
|
||||
await session.execute(
|
||||
select(ProjectShare).where(
|
||||
ProjectShare.project_id == project_id,
|
||||
ProjectShare.shared_with_user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
# Group shares
|
||||
group_ids = await _user_group_ids(session, user_id)
|
||||
group_perm: str | None = None
|
||||
if group_ids:
|
||||
group_shares = (
|
||||
await session.execute(
|
||||
select(ProjectShare).where(
|
||||
ProjectShare.project_id == project_id,
|
||||
ProjectShare.shared_with_group_id.in_(group_ids),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
for gs in group_shares:
|
||||
group_perm = _higher(group_perm, gs.permission)
|
||||
|
||||
return _higher(direct.permission if direct else None, group_perm)
|
||||
|
||||
|
||||
async def can_read_project(user_id: int, project_id: int) -> bool:
|
||||
return (await get_project_permission(user_id, project_id)) is not None
|
||||
|
||||
|
||||
async def can_write_project(user_id: int, project_id: int) -> bool:
|
||||
perm = await get_project_permission(user_id, project_id)
|
||||
return perm in ("editor", "admin", "owner")
|
||||
|
||||
|
||||
async def can_admin_project(user_id: int, project_id: int) -> bool:
|
||||
perm = await get_project_permission(user_id, project_id)
|
||||
return perm in ("admin", "owner")
|
||||
|
||||
|
||||
async def is_instance_admin(user_id: int) -> bool:
|
||||
"""Whether the user administers the INSTANCE (users.role == "admin") —
|
||||
the gate on the global catalogs (canonical areas, platforms), which belong
|
||||
to no user and no project, so no share can grant a write to them."""
|
||||
async with async_session() as session:
|
||||
role = await session.scalar(select(User.role).where(User.id == user_id))
|
||||
return role == "admin"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Note / task permissions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_note_permission(user_id: int, note_id: int) -> str | None:
|
||||
"""Return the effective permission for user on a note/task, or None.
|
||||
|
||||
Resolution order:
|
||||
1. Ownership
|
||||
2. Direct note share
|
||||
3. Group-based note share
|
||||
4. Inherited from project share (if note belongs to a project)
|
||||
Highest rank wins.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
note = await session.get(Note, note_id)
|
||||
if note is None:
|
||||
return None
|
||||
if note.user_id == user_id:
|
||||
return "owner"
|
||||
|
||||
direct = (
|
||||
await session.execute(
|
||||
select(NoteShare).where(
|
||||
NoteShare.note_id == note_id,
|
||||
NoteShare.shared_with_user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
group_ids = await _user_group_ids(session, user_id)
|
||||
group_perm: str | None = None
|
||||
if group_ids:
|
||||
group_shares = (
|
||||
await session.execute(
|
||||
select(NoteShare).where(
|
||||
NoteShare.note_id == note_id,
|
||||
NoteShare.shared_with_group_id.in_(group_ids),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
for gs in group_shares:
|
||||
group_perm = _higher(group_perm, gs.permission)
|
||||
|
||||
note_perm = _higher(direct.permission if direct else None, group_perm)
|
||||
|
||||
# Inherit from project if note belongs to one
|
||||
if note.project_id is not None:
|
||||
project_perm = await get_project_permission(user_id, note.project_id)
|
||||
note_perm = _higher(note_perm, project_perm)
|
||||
|
||||
return note_perm
|
||||
|
||||
|
||||
async def can_read_note(user_id: int, note_id: int) -> bool:
|
||||
return (await get_note_permission(user_id, note_id)) is not None
|
||||
|
||||
|
||||
async def can_write_note(user_id: int, note_id: int) -> bool:
|
||||
perm = await get_note_permission(user_id, note_id)
|
||||
return perm in ("editor", "admin", "owner")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Design-system permissions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_design_system_permission(
|
||||
user_id: int, design_system_id: int
|
||||
) -> str | None:
|
||||
"""Effective permission on a design system, or None.
|
||||
|
||||
Two ways in, and the asymmetry between them is the point:
|
||||
|
||||
- **Owning it** grants "owner" — full read and write.
|
||||
- **Reaching it through a project you can see** grants "viewer", and only
|
||||
ever "viewer". Being an editor on a shared project must NOT confer the
|
||||
right to rewrite the family system that project inherits from: one
|
||||
project's collaborator would be editing tokens every other project in
|
||||
the family resolves through. Editing a design system stays the owner's
|
||||
act, and it is the same reasoning that keeps a rulebook owner-scoped.
|
||||
|
||||
Reachability follows the parent chain UPWARD. Rendering a project's UI means
|
||||
resolving its whole chain, so read access to a system implies read access to
|
||||
its ancestors — otherwise a shared project would resolve to a truncated
|
||||
cascade and silently render with the wrong values.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
if system.owner_user_id == user_id:
|
||||
return "owner"
|
||||
|
||||
shared_project_ids = select(ProjectShare.project_id).where(
|
||||
or_(
|
||||
ProjectShare.shared_with_user_id == user_id,
|
||||
ProjectShare.shared_with_group_id.in_(_my_group_ids(user_id)),
|
||||
)
|
||||
)
|
||||
entry_points = set(
|
||||
(
|
||||
await session.execute(
|
||||
select(Project.design_system_id).where(
|
||||
Project.design_system_id.is_not(None),
|
||||
Project.deleted_at.is_(None),
|
||||
or_(
|
||||
Project.user_id == user_id,
|
||||
Project.id.in_(shared_project_ids),
|
||||
),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
if not entry_points:
|
||||
return None
|
||||
|
||||
# One narrow query for the whole forest's shape. Design systems are a
|
||||
# handful of rows per install — a family and one per app — so walking
|
||||
# from each entry point in memory beats a recursive CTE per check.
|
||||
parents = dict(
|
||||
(
|
||||
await session.execute(
|
||||
select(DesignSystem.id, DesignSystem.parent_id).where(
|
||||
DesignSystem.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
for entry in entry_points:
|
||||
if design_system_id in ancestry(entry, parents):
|
||||
return "viewer"
|
||||
return None
|
||||
|
||||
|
||||
async def can_read_design_system(user_id: int, design_system_id: int) -> bool:
|
||||
return (await get_design_system_permission(user_id, design_system_id)) is not None
|
||||
|
||||
|
||||
async def can_write_design_system(user_id: int, design_system_id: int) -> bool:
|
||||
perm = await get_design_system_permission(user_id, design_system_id)
|
||||
return perm in ("editor", "admin", "owner")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Set-based visibility (for LIST queries)
|
||||
#
|
||||
# Two scopes, deliberately different:
|
||||
#
|
||||
# readable_* — everything the ACL permits, including a record reachable ONLY
|
||||
# through a direct or group note share. For EXPLICIT acts: a
|
||||
# search the caller typed, or a fetch by id.
|
||||
# browsable_* — the caller's own records plus anything in a project they have
|
||||
# access to. For PASSIVE surfaces: browse lists, facet counts,
|
||||
# the process→skill manifest.
|
||||
#
|
||||
# The split is a trust boundary, not an optimisation. Anything that appears
|
||||
# unasked — in your own list, your own counts, or as a skill installed on your
|
||||
# machine — reads as material you endorsed. A one-off record someone shared with
|
||||
# you has not earned that standing, so it waits until you go looking for it.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def notes_visibility_clause(user_id: int, scope: str = "own"):
|
||||
"""The one place a retrieval declares how far it may see.
|
||||
|
||||
Every path that returns notes picks a scope by what KIND of act it is:
|
||||
|
||||
"own" — the caller's records only. For machinery whose answer must not
|
||||
depend on other people: the near-duplicate gate can't block a
|
||||
create because a stranger wrote something similar, and can't
|
||||
point at a record the caller cannot edit.
|
||||
"browse" — own + project-reachable. For passive surfaces, where an
|
||||
unrequested record would read as endorsed.
|
||||
"read" — everything the ACL permits. For explicit acts: a typed search,
|
||||
a fetch by id.
|
||||
|
||||
Defaults to the narrowest, so a new caller that forgets to choose is wrong in
|
||||
the safe direction.
|
||||
"""
|
||||
if scope == "own":
|
||||
return Note.user_id == user_id
|
||||
if scope == "browse":
|
||||
return browsable_notes_clause(user_id)
|
||||
if scope == "read":
|
||||
return readable_notes_clause(user_id)
|
||||
raise ValueError(f"unknown note scope {scope!r} (own | browse | read)")
|
||||
|
||||
|
||||
async def owner_names_for(user_ids: set[int]) -> dict[int, str]:
|
||||
"""{user_id: username} in one query. Empty input costs nothing.
|
||||
|
||||
Fails soft: on a lookup error the caller gets no names and renders "another
|
||||
user" instead. Losing an attribution is a cosmetic downgrade, and the part
|
||||
that matters — that the record ISN'T the caller's — comes from comparing
|
||||
owner ids, not from this. Failing the whole search over a username would be
|
||||
the worse trade.
|
||||
"""
|
||||
if not user_ids:
|
||||
return {}
|
||||
try:
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(User.id, User.username).where(User.id.in_(user_ids))
|
||||
)
|
||||
).all()
|
||||
return {uid: name for uid, name in rows}
|
||||
except Exception:
|
||||
logger.warning("Owner-name lookup failed; falling back to unnamed "
|
||||
"attribution", exc_info=True)
|
||||
return {}
|
||||
|
||||
|
||||
def _my_group_ids(user_id: int):
|
||||
"""The caller's group ids as a SUBQUERY, not a fetched list.
|
||||
|
||||
Keeping it in SQL is what makes the clause builders below pure functions —
|
||||
no session, no await, nothing for a caller's unit test to mock — and it folds
|
||||
the membership lookup into the one statement the caller was already running.
|
||||
"""
|
||||
return select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
||||
|
||||
|
||||
def readable_notes_clause(user_id: int):
|
||||
"""A SQLAlchemy WHERE predicate matching every note this user may READ.
|
||||
|
||||
`get_note_permission` answers "may I read THIS note?" one row at a time,
|
||||
which a list query can't use — checking per row is O(n) round-trips. This is
|
||||
the same resolution expressed as set membership so it can go straight into a
|
||||
`select(Note).where(...)`:
|
||||
|
||||
1. ownership
|
||||
2. a direct note share
|
||||
3. a group note share
|
||||
4. inheritance from a shared project
|
||||
|
||||
Keep the two in step: a rule added here belongs in `get_note_permission`
|
||||
too, or a note becomes findable but not openable (or the reverse — which is
|
||||
the bug this was written to fix).
|
||||
"""
|
||||
groups = _my_group_ids(user_id)
|
||||
|
||||
shared_note_ids = select(NoteShare.note_id).where(
|
||||
or_(
|
||||
NoteShare.shared_with_user_id == user_id,
|
||||
NoteShare.shared_with_group_id.in_(groups),
|
||||
)
|
||||
)
|
||||
shared_project_ids = select(ProjectShare.project_id).where(
|
||||
or_(
|
||||
ProjectShare.shared_with_user_id == user_id,
|
||||
ProjectShare.shared_with_group_id.in_(groups),
|
||||
)
|
||||
)
|
||||
|
||||
return or_(
|
||||
Note.user_id == user_id,
|
||||
Note.id.in_(shared_note_ids),
|
||||
Note.project_id.in_(shared_project_ids),
|
||||
)
|
||||
|
||||
|
||||
async def describe_provenance(user_id: int, note) -> dict:
|
||||
"""Provenance for a record being handed to a caller: `{}` when it's theirs,
|
||||
otherwise `{"shared": True, "owner": <username>, "permission": <perm>}`.
|
||||
|
||||
Anything reaching an agent or a UI from another person has to say so. A
|
||||
shared record is one person's suggestion, not a standard the caller adopted,
|
||||
and without a marker the two are indistinguishable — the reader would assume
|
||||
their own past self wrote it and treat it as settled practice.
|
||||
"""
|
||||
if note is None or note.user_id == user_id:
|
||||
return {}
|
||||
async with async_session() as session:
|
||||
owner = await session.get(User, note.user_id)
|
||||
return {
|
||||
"shared": True,
|
||||
"owner": owner.username if owner else None,
|
||||
"permission": await get_note_permission(user_id, note.id),
|
||||
}
|
||||
|
||||
|
||||
async def label_shared_items(user_id: int, items: list[dict]) -> list[dict]:
|
||||
"""Mark the entries in a list payload that belong to someone else.
|
||||
|
||||
Each foreign item gains `shared: True` and `owner: <username>`; the caller's
|
||||
own items are left untouched so an all-mine list stays noise-free. Usernames
|
||||
are resolved in one query per distinct owner, not one per row.
|
||||
|
||||
Lists are where provenance matters most: an unmarked row in your own list
|
||||
reads as something you recorded and vetted.
|
||||
"""
|
||||
foreign = {
|
||||
it["user_id"] for it in items
|
||||
if it.get("user_id") is not None and it["user_id"] != user_id
|
||||
}
|
||||
if not foreign:
|
||||
return items
|
||||
names = await owner_names_for(foreign)
|
||||
for it in items:
|
||||
owner_id = it.get("user_id")
|
||||
if owner_id is not None and owner_id != user_id:
|
||||
it["shared"] = True
|
||||
it["owner"] = names.get(owner_id)
|
||||
return items
|
||||
|
||||
|
||||
def browsable_notes_clause(user_id: int):
|
||||
"""A WHERE predicate for PASSIVE surfaces: the caller's own notes, plus
|
||||
notes in a project they can reach (owned or shared).
|
||||
|
||||
Deliberately narrower than `readable_notes_clause` — it omits records the
|
||||
caller can read *only* via a direct or group note share. Those stay
|
||||
search-only, so a one-off someone handed them never drifts into a browse
|
||||
list, a facet count, or the skill manifest as though it were their own.
|
||||
|
||||
Project membership is the seam because it's a standing, mutual context: if
|
||||
you're on the project, its content is your working material. A note with no
|
||||
project that you don't own therefore never matches — `NULL IN (...)` is not
|
||||
true — which is exactly the intent.
|
||||
"""
|
||||
shared_project_ids = select(ProjectShare.project_id).where(
|
||||
or_(
|
||||
ProjectShare.shared_with_user_id == user_id,
|
||||
ProjectShare.shared_with_group_id.in_(_my_group_ids(user_id)),
|
||||
)
|
||||
)
|
||||
owned_project_ids = select(Project.id).where(Project.user_id == user_id)
|
||||
|
||||
return or_(
|
||||
Note.user_id == user_id,
|
||||
Note.project_id.in_(shared_project_ids),
|
||||
Note.project_id.in_(owned_project_ids),
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.api_key import ApiKey
|
||||
|
||||
|
||||
def generate_key() -> str:
|
||||
"""Generate a new full API key. Never stored — caller must hash it."""
|
||||
return "fmcp_" + secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(raw: str) -> str:
|
||||
"""The ONE fingerprint for every bearer secret stored by hash — API keys,
|
||||
password-reset tokens, invitation tokens. Stored rows hold this, never
|
||||
the raw value; a lookup hashes the presented token and compares."""
|
||||
return hashlib.sha256(raw.encode()).hexdigest()
|
||||
|
||||
|
||||
def _key_prefix(key: str) -> str:
|
||||
"""Return first 12 chars of the key for display (e.g. 'fmcp_abcdefg')."""
|
||||
return key[:12]
|
||||
|
||||
|
||||
async def create_api_key(
|
||||
user_id: int, name: str, scope: str
|
||||
) -> tuple[str, dict]:
|
||||
"""Create a new API key. Returns (full_key, key_dict). full_key is never stored."""
|
||||
if scope not in ("read", "write"):
|
||||
raise ValueError("scope must be 'read' or 'write'")
|
||||
full_key = generate_key()
|
||||
key = ApiKey(
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
key_hash=hash_token(full_key),
|
||||
key_prefix=_key_prefix(full_key),
|
||||
scope=scope,
|
||||
)
|
||||
async with async_session() as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await session.refresh(key)
|
||||
return full_key, key.to_dict()
|
||||
|
||||
|
||||
async def list_api_keys(user_id: int) -> list[dict]:
|
||||
"""List all non-revoked API keys for the user."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(ApiKey)
|
||||
.where(ApiKey.user_id == user_id, ApiKey.revoked_at.is_(None))
|
||||
.order_by(ApiKey.created_at.desc())
|
||||
)
|
||||
return [k.to_dict() for k in result.scalars().all()]
|
||||
|
||||
|
||||
async def revoke_api_key(user_id: int, key_id: int) -> bool:
|
||||
"""Soft-delete a key by setting revoked_at. Returns True if found."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(ApiKey).where(ApiKey.id == key_id, ApiKey.user_id == user_id)
|
||||
)
|
||||
key = result.scalars().first()
|
||||
if key is None:
|
||||
return False
|
||||
key.revoked_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def lookup_key(raw_key: str) -> ApiKey | None:
|
||||
"""Look up a non-revoked ApiKey by raw token value. Updates last_used_at."""
|
||||
key_hash = hash_token(raw_key)
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.key_hash == key_hash,
|
||||
ApiKey.revoked_at.is_(None),
|
||||
)
|
||||
)
|
||||
key = result.scalars().first()
|
||||
if key is None:
|
||||
return None
|
||||
key.last_used_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(key)
|
||||
return key
|
||||
@@ -0,0 +1,405 @@
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import bcrypt
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.invitation import InvitationToken
|
||||
from grimoire.models.password_reset import PasswordResetToken
|
||||
from grimoire.models.setting import Setting
|
||||
from grimoire.models.user import User
|
||||
from grimoire.services.api_keys import hash_token
|
||||
from grimoire.services.settings import get_admin_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
return bcrypt.checkpw(password.encode(), password_hash.encode())
|
||||
|
||||
|
||||
async def get_user_count() -> int:
|
||||
async with async_session() as session:
|
||||
return await session.scalar(select(func.count(User.id))) or 0
|
||||
|
||||
|
||||
async def create_user(
|
||||
username: str,
|
||||
password: str | None,
|
||||
email: str | None = None,
|
||||
oauth_sub: str | None = None,
|
||||
) -> User:
|
||||
user_count = await get_user_count()
|
||||
role = "admin" if user_count == 0 else "user"
|
||||
|
||||
async with async_session() as session:
|
||||
user = User(
|
||||
username=username,
|
||||
email=email,
|
||||
password_hash=hash_password(password) if password is not None else None,
|
||||
oauth_sub=oauth_sub,
|
||||
role=role,
|
||||
)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
|
||||
# First user claims all pre-existing data (migration assigns user_id=1,
|
||||
# which matches SERIAL first insert; also handles any NULLs)
|
||||
if role == "admin":
|
||||
await session.execute(
|
||||
update(Note)
|
||||
.where((Note.user_id.is_(None)) | (Note.user_id == user.id))
|
||||
.values(user_id=user.id)
|
||||
)
|
||||
await session.execute(
|
||||
update(Setting)
|
||||
.where(Setting.user_id.is_(None))
|
||||
.values(user_id=user.id)
|
||||
)
|
||||
# Auto-close registration after first user setup
|
||||
session.add(Setting(user_id=user.id, key="registration_open", value="false"))
|
||||
await session.commit()
|
||||
logger.info("First user '%s' created as admin, claimed orphaned data", username)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
async def authenticate(username: str, password: str) -> User | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(User).where(User.username == username)
|
||||
)
|
||||
user = result.scalars().first()
|
||||
if user is None:
|
||||
return None
|
||||
if user.password_hash is None:
|
||||
# OAuth-only user — cannot use password login
|
||||
return None
|
||||
if verify_password(password, user.password_hash):
|
||||
return user
|
||||
return None
|
||||
|
||||
|
||||
async def get_user_by_id(user_id: int) -> User | None:
|
||||
async with async_session() as session:
|
||||
return await session.get(User, user_id)
|
||||
|
||||
|
||||
async def get_user_by_username(username: str) -> User | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(User).where(User.username == username)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def change_password(user_id: int, current_password: str, new_password: str) -> int | None:
|
||||
"""Change a user's password. Returns new session_version on success, None if current password is wrong."""
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, user_id)
|
||||
if not user:
|
||||
return None
|
||||
# OAuth-only accounts have no local password hash — verify_password
|
||||
# would crash on None. Treat as "no local credential to change" (the
|
||||
# route turns None into a clean 4xx, not a 500).
|
||||
if user.password_hash is None:
|
||||
return None
|
||||
if not verify_password(current_password, user.password_hash):
|
||||
return None
|
||||
user.password_hash = hash_password(new_password)
|
||||
user.session_version += 1
|
||||
await session.commit()
|
||||
logger.info("Password changed for user %d (%s)", user_id, user.username)
|
||||
return user.session_version
|
||||
|
||||
|
||||
async def invalidate_other_sessions(user_id: int) -> int:
|
||||
"""Bump session_version so all sessions except the current one are invalidated.
|
||||
|
||||
Returns the new session_version so the caller can update the active session cookie.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, user_id)
|
||||
user.session_version += 1
|
||||
await session.commit()
|
||||
logger.info("Sessions invalidated for user %d (%s)", user_id, user.username)
|
||||
return user.session_version
|
||||
|
||||
|
||||
async def is_registration_open() -> bool:
|
||||
"""Check if new user registration is allowed.
|
||||
|
||||
Always open when no users exist (first-user setup).
|
||||
Otherwise reads the admin's 'registration_open' setting (default closed).
|
||||
"""
|
||||
user_count = await get_user_count()
|
||||
if user_count == 0:
|
||||
return True
|
||||
return await get_admin_setting("registration_open", "false") == "true"
|
||||
|
||||
|
||||
async def list_users() -> list[User]:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(select(User).order_by(User.created_at))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def delete_user(user_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, user_id)
|
||||
if not user:
|
||||
return False
|
||||
await session.delete(user)
|
||||
await session.commit()
|
||||
logger.info("Deleted user %d (%s)", user_id, user.username)
|
||||
return True
|
||||
|
||||
|
||||
async def set_registration_open(admin_user_id: int, open: bool) -> None:
|
||||
from grimoire.services.settings import set_setting
|
||||
await set_setting(admin_user_id, "registration_open", "true" if open else "false")
|
||||
|
||||
|
||||
async def update_user_email(user_id: int, email: str | None) -> None:
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, user_id)
|
||||
if user:
|
||||
user.email = email
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_user_by_oauth_sub(sub: str) -> User | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(User).where(User.oauth_sub == sub)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def link_oauth_sub(user_id: int, sub: str) -> None:
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
update(User).where(User.id == user_id).values(oauth_sub=sub)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_user_by_email(email: str) -> User | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(User).where(func.lower(User.email) == email.lower())
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def create_password_reset_token(user_id: int) -> str:
|
||||
"""Generate a password reset token. Returns the raw token (for the email link)."""
|
||||
raw_token = secrets.token_urlsafe(32)
|
||||
token_hash = hash_token(raw_token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(hours=1)
|
||||
|
||||
async with async_session() as session:
|
||||
# Invalidate any existing unused tokens for this user
|
||||
result = await session.execute(
|
||||
select(PasswordResetToken).where(
|
||||
PasswordResetToken.user_id == user_id,
|
||||
PasswordResetToken.used.is_(False),
|
||||
)
|
||||
)
|
||||
for old_token in result.scalars().all():
|
||||
old_token.used = True
|
||||
|
||||
token = PasswordResetToken(
|
||||
user_id=user_id,
|
||||
token_hash=token_hash,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
session.add(token)
|
||||
await session.commit()
|
||||
|
||||
logger.info("Password reset token created for user %d", user_id)
|
||||
return raw_token
|
||||
|
||||
|
||||
async def reset_password_with_token(raw_token: str, new_password: str) -> int | None:
|
||||
"""Validate a reset token and update the user's password. Returns user_id on success."""
|
||||
token_hash = hash_token(raw_token)
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(PasswordResetToken).where(PasswordResetToken.token_hash == token_hash)
|
||||
)
|
||||
reset_token = result.scalars().first()
|
||||
|
||||
if not reset_token:
|
||||
return None
|
||||
if reset_token.used:
|
||||
return None
|
||||
if reset_token.expires_at < datetime.now(timezone.utc):
|
||||
return None
|
||||
|
||||
user = await session.get(User, reset_token.user_id)
|
||||
if not user:
|
||||
return None
|
||||
|
||||
user.password_hash = hash_password(new_password)
|
||||
user.session_version += 1
|
||||
reset_token.used = True
|
||||
await session.commit()
|
||||
|
||||
logger.info("Password reset via token for user %d (%s)", user.id, user.username)
|
||||
return user.id
|
||||
|
||||
|
||||
async def create_invitation(email: str, invited_by: int) -> str:
|
||||
"""Generate an invitation token. Returns the raw token (for the email link)."""
|
||||
raw_token = secrets.token_urlsafe(32)
|
||||
token_hash = hash_token(raw_token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(days=7)
|
||||
|
||||
async with async_session() as session:
|
||||
# Invalidate previous unused invitations for same email
|
||||
result = await session.execute(
|
||||
select(InvitationToken).where(
|
||||
func.lower(InvitationToken.email) == email.lower(),
|
||||
InvitationToken.used.is_(False),
|
||||
)
|
||||
)
|
||||
for old_token in result.scalars().all():
|
||||
old_token.used = True
|
||||
|
||||
token = InvitationToken(
|
||||
email=email.lower(),
|
||||
token_hash=token_hash,
|
||||
invited_by=invited_by,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
session.add(token)
|
||||
await session.commit()
|
||||
|
||||
logger.info("Invitation created for %s by user %d", email, invited_by)
|
||||
return raw_token
|
||||
|
||||
|
||||
async def validate_invitation_token(raw_token: str) -> InvitationToken | None:
|
||||
"""Look up by hash, check not used/expired. Returns the token record with email."""
|
||||
token_hash = hash_token(raw_token)
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(InvitationToken).where(InvitationToken.token_hash == token_hash)
|
||||
)
|
||||
invitation = result.scalars().first()
|
||||
|
||||
if not invitation:
|
||||
return None
|
||||
if invitation.used:
|
||||
return None
|
||||
if invitation.expires_at < datetime.now(timezone.utc):
|
||||
return None
|
||||
|
||||
return invitation
|
||||
|
||||
|
||||
async def register_with_invitation(raw_token: str, username: str, password: str) -> User | None:
|
||||
"""Validate token, create user with the invitation's email, mark token used."""
|
||||
token_hash = hash_token(raw_token)
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(InvitationToken).where(InvitationToken.token_hash == token_hash)
|
||||
)
|
||||
invitation = result.scalars().first()
|
||||
|
||||
if not invitation or invitation.used or invitation.expires_at < datetime.now(timezone.utc):
|
||||
return None
|
||||
|
||||
invitation_id = invitation.id
|
||||
invite_email = invitation.email
|
||||
|
||||
# Create the user FIRST, then consume the token. create_user can fail (e.g.
|
||||
# a username collision raises and the route returns 409); marking the token
|
||||
# used before that would burn this single-use invite and lock the invitee
|
||||
# out of retrying. Ordering it after means a failed creation leaves the
|
||||
# token valid.
|
||||
user = await create_user(username, password, invite_email)
|
||||
|
||||
async with async_session() as session:
|
||||
invitation = await session.get(InvitationToken, invitation_id)
|
||||
if invitation is not None:
|
||||
invitation.used = True
|
||||
await session.commit()
|
||||
|
||||
logger.info("User '%s' registered via invitation for %s", username, invite_email)
|
||||
return user
|
||||
|
||||
|
||||
async def list_pending_invitations() -> list[InvitationToken]:
|
||||
"""List unused, non-expired invitations."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(InvitationToken).where(
|
||||
InvitationToken.used.is_(False),
|
||||
InvitationToken.expires_at > datetime.now(timezone.utc),
|
||||
).order_by(InvitationToken.created_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def revoke_invitation(invitation_id: int) -> bool:
|
||||
"""Mark an invitation as used (revoked)."""
|
||||
async with async_session() as session:
|
||||
invitation = await session.get(InvitationToken, invitation_id)
|
||||
if not invitation or invitation.used:
|
||||
return False
|
||||
invitation.used = True
|
||||
await session.commit()
|
||||
logger.info("Invitation %d revoked", invitation_id)
|
||||
return True
|
||||
|
||||
|
||||
# ── Token retention ─────────────────────────────────────────────────────
|
||||
# Password-reset and invitation tokens are only ever flipped used=True and are
|
||||
# never pruned, so on a long-lived instance both tables grow without bound.
|
||||
# A daily sweep deletes any token whose validity window ended over grace_days
|
||||
# ago (covers both used and naturally-expired rows once they're cold).
|
||||
|
||||
_auth_retention_task = None
|
||||
|
||||
|
||||
async def purge_expired_auth_tokens(grace_days: int = 7) -> int:
|
||||
"""Delete password-reset / invitation tokens that expired > grace_days ago."""
|
||||
from sqlalchemy import delete
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=grace_days)
|
||||
removed = 0
|
||||
async with async_session() as session:
|
||||
for model in (PasswordResetToken, InvitationToken):
|
||||
result = await session.execute(
|
||||
delete(model).where(model.expires_at < cutoff)
|
||||
)
|
||||
removed += result.rowcount or 0
|
||||
await session.commit()
|
||||
return removed
|
||||
|
||||
|
||||
async def _auth_token_retention_tick() -> None:
|
||||
removed = await purge_expired_auth_tokens()
|
||||
if removed:
|
||||
logger.info("Auth token retention: deleted %d expired token(s)", removed)
|
||||
|
||||
|
||||
def start_auth_token_retention_loop() -> None:
|
||||
global _auth_retention_task
|
||||
if _auth_retention_task is None or _auth_retention_task.done():
|
||||
from grimoire.services.background import start_periodic
|
||||
_auth_retention_task = start_periodic(
|
||||
86400, _auth_token_retention_tick, label="auth_token_retention", # daily
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Fire-and-forget background tasks that actually run.
|
||||
|
||||
The event loop holds only a WEAK reference to a task, so a bare
|
||||
``create_task`` with no other holder can be garbage-collected mid-flight — a
|
||||
write that never errors and never lands. This module is
|
||||
the one place that gets the pattern right: strong references in ``_pending``,
|
||||
discarded on completion, with failures logged at WARNING instead of vanishing.
|
||||
|
||||
``retrieval_telemetry`` predates this module and keeps its own copy, because
|
||||
its canary is a genuinely different shape — one process-wide flag and no
|
||||
AppLog row. ``note_usage`` and ``rule_usage`` share ``report_telemetry_failure``
|
||||
below. New fire-and-forget callers use ``spawn`` rather than writing another
|
||||
copy of the strong-reference dance.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import traceback
|
||||
from collections.abc import Coroutine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_pending: set[asyncio.Task] = set()
|
||||
|
||||
# Sites that have already dropped their once-per-process AppLog row, keyed
|
||||
# "<subsystem>:<site>". A readout can run on every list render — without this,
|
||||
# a broken table turns the error log into a firehose that buries the finding it
|
||||
# exists to surface.
|
||||
_reported: set[str] = set()
|
||||
|
||||
|
||||
async def report_telemetry_failure(subsystem: str, site: str) -> None:
|
||||
"""Make a swallowed telemetry failure visible. Call from an except block.
|
||||
|
||||
WARNING to the process log every time; one AppLog error row per process per
|
||||
(subsystem, site) so the admin UI shows the outage without host access.
|
||||
|
||||
THIS IS NOT DECORATION. A telemetry subsystem whose every failure goes to
|
||||
``logger.debug`` can run at zero — every counter reading empty,
|
||||
indistinguishable from "nobody uses this". A subsystem whose failures are
|
||||
all invisible cannot report its own death.
|
||||
|
||||
The AppLog write is itself guarded: when the database is down it fails too,
|
||||
and that is fine. The WARNING already said so, and a canary must never take
|
||||
down the surface it watches.
|
||||
"""
|
||||
logger.warning("%s telemetry %s failed", subsystem, site, exc_info=True)
|
||||
key = f"{subsystem}:{site}"
|
||||
if key in _reported:
|
||||
return
|
||||
_reported.add(key)
|
||||
try:
|
||||
from grimoire.services.logging import log_error
|
||||
|
||||
await log_error(
|
||||
endpoint=subsystem,
|
||||
error_type=f"{subsystem}_{site}_failed",
|
||||
error_message=f"{subsystem} telemetry {site} is failing; "
|
||||
"usage counters will read zero until this is fixed",
|
||||
traceback=traceback.format_exc(),
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("%s canary write failed", subsystem, exc_info=True)
|
||||
|
||||
|
||||
def spawn(coro: Coroutine, *, site: str) -> None:
|
||||
"""Schedule ``coro`` fire-and-forget; ``site`` names it in failure logs.
|
||||
|
||||
No running loop (sync context outside the app) closes the coroutine and
|
||||
skips — every app path runs on the loop, and blocking would be worse.
|
||||
"""
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(coro)
|
||||
except RuntimeError:
|
||||
coro.close()
|
||||
logger.debug("background task %s skipped — no running event loop", site)
|
||||
return
|
||||
_pending.add(task)
|
||||
|
||||
def _done(t: asyncio.Task) -> None:
|
||||
_pending.discard(t)
|
||||
if not t.cancelled() and t.exception() is not None:
|
||||
logger.warning(
|
||||
"background task %s failed", site, exc_info=t.exception()
|
||||
)
|
||||
|
||||
task.add_done_callback(_done)
|
||||
|
||||
|
||||
def start_periodic(interval_s: float, work, *, label: str) -> asyncio.Task:
|
||||
"""A forever loop that sleeps ``interval_s`` then awaits ``work()``, logging
|
||||
(never raising) when a tick fails — the one shape the hourly/daily
|
||||
retention sweeps share (log retention, notification sweep, auth-token
|
||||
purge). Sleeps FIRST so startup isn't a sweep; holds a strong reference
|
||||
like spawn() so the loop cannot be garbage-collected mid-flight."""
|
||||
async def _loop() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(interval_s)
|
||||
try:
|
||||
await work()
|
||||
except Exception:
|
||||
logger.exception("periodic task %s failed", label)
|
||||
|
||||
task = asyncio.get_running_loop().create_task(_loop(), name=f"periodic-{label}")
|
||||
_pending.add(task)
|
||||
task.add_done_callback(_pending.discard)
|
||||
return task
|
||||
|
||||
|
||||
async def drain() -> None:
|
||||
"""Await everything in flight — for tests that need the writes landed."""
|
||||
while _pending:
|
||||
await asyncio.gather(*list(_pending), return_exceptions=True)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,262 @@
|
||||
"""Where the scheduled backup goes, and getting a backup file there.
|
||||
|
||||
Two kinds of destination, both instance-wide and set by an admin:
|
||||
|
||||
- **folder**: a directory inside the container, usually a mounted share. A
|
||||
backup service that watches the share (or a person) takes it from there.
|
||||
- **s3**: any S3-compatible bucket, addressed by path. Grimoire only puts and
|
||||
lists, so an append-only key is enough.
|
||||
|
||||
Either way Grimoire ADDS one dated file per run and never removes one: keeping
|
||||
and thinning the history is the destination's job (services/backup_file.py
|
||||
says why the file is shaped the way it is for that).
|
||||
|
||||
The settings live as admin settings: `backup_config` holds everything but the
|
||||
S3 secret as JSON, and `backup_s3_secret` holds the secret sealed under the
|
||||
instance's SECRET_KEY (services/secrets.py). The secret is also kept out of
|
||||
every backup (backup._SETTINGS_NOT_EXPORTED), so a backup copied off-site
|
||||
does not carry the key that writes to that very place.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import asdict, dataclass, fields, replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from grimoire.services import backup_file
|
||||
from grimoire.services.backup_s3 import S3Error, S3Target, list_objects, put_file
|
||||
from grimoire.services.secrets import seal, unseal
|
||||
from grimoire.services.settings import get_admin_setting, set_admin_setting
|
||||
|
||||
CONFIG_SETTING = "backup_config"
|
||||
SECRET_SETTING = "backup_s3_secret"
|
||||
DESTINATION_KINDS = ("folder", "s3")
|
||||
DEFAULT_FOLDER = "/backups"
|
||||
# 02:00 UTC: before the 03:30 trash purge and the 04:00 DB maintenance, so the
|
||||
# backup never contends with a VACUUM, and the night's deletions are still
|
||||
# recoverable from the backup taken just before them.
|
||||
DEFAULT_HOUR = 2
|
||||
|
||||
|
||||
class DestinationError(Exception):
|
||||
"""Why the destination can't take a backup, said to the person."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupConfig:
|
||||
enabled: bool = False
|
||||
hour: int = DEFAULT_HOUR
|
||||
destination: str = "folder"
|
||||
folder_path: str = DEFAULT_FOLDER
|
||||
s3_endpoint: str = ""
|
||||
s3_bucket: str = ""
|
||||
s3_access_key: str = ""
|
||||
s3_region: str = "us-east-1"
|
||||
s3_verify_tls: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict) -> "BackupConfig":
|
||||
"""Lenient: unknown keys are ignored and bad values fall back to the
|
||||
default, so an edited or older stored config still loads."""
|
||||
known = {f.name: f for f in fields(cls)}
|
||||
values = {}
|
||||
for name, value in (raw or {}).items():
|
||||
f = known.get(name)
|
||||
if f is None:
|
||||
continue
|
||||
default = f.default
|
||||
if isinstance(default, bool):
|
||||
values[name] = value if isinstance(value, bool) else default
|
||||
elif isinstance(default, int):
|
||||
try:
|
||||
values[name] = int(value)
|
||||
except (TypeError, ValueError):
|
||||
values[name] = default
|
||||
else:
|
||||
values[name] = str(value).strip() if value is not None else default
|
||||
config = cls(**values)
|
||||
if not 0 <= config.hour <= 23:
|
||||
config = replace(config, hour=DEFAULT_HOUR)
|
||||
if config.destination not in DESTINATION_KINDS:
|
||||
config = replace(config, destination="folder")
|
||||
return config
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
def problems(self, *, has_secret: bool) -> list[str]:
|
||||
"""What stops this config from sending a backup, in words. Empty when
|
||||
it is complete (which says nothing about whether the place answers —
|
||||
check_destination does that)."""
|
||||
out: list[str] = []
|
||||
if self.destination == "folder":
|
||||
if not self.folder_path.startswith("/"):
|
||||
out.append("The folder must be an absolute path inside the container.")
|
||||
else:
|
||||
if not self.s3_endpoint.startswith(("http://", "https://")):
|
||||
out.append("The S3 address must start with http:// or https://.")
|
||||
if not self.s3_bucket:
|
||||
out.append("The bucket name is missing.")
|
||||
if not self.s3_access_key:
|
||||
out.append("The access key is missing.")
|
||||
if not has_secret:
|
||||
out.append("The secret key is missing.")
|
||||
return out
|
||||
|
||||
|
||||
def apply_changes(current: BackupConfig, data: dict) -> BackupConfig:
|
||||
"""`current` with the fields an admin sent changed, checked strictly.
|
||||
|
||||
`from_dict` forgives a stored value it can't read; a value a person just
|
||||
typed is refused instead, with the reason, so a typo in the hour does not
|
||||
silently become 02:00. Keys that aren't config fields (the secret, say)
|
||||
are ignored here.
|
||||
"""
|
||||
names = {f.name for f in fields(BackupConfig)}
|
||||
changes = {k: v for k, v in (data or {}).items() if k in names}
|
||||
if "hour" in changes:
|
||||
try:
|
||||
hour = int(changes["hour"])
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError("The hour must be a whole number from 0 to 23.") from None
|
||||
if not 0 <= hour <= 23:
|
||||
raise ValueError("The hour must be from 0 to 23 (UTC).")
|
||||
changes["hour"] = hour
|
||||
if "destination" in changes and changes["destination"] not in DESTINATION_KINDS:
|
||||
raise ValueError("The destination must be a folder or an S3 bucket.")
|
||||
for name in ("enabled", "s3_verify_tls"):
|
||||
if name in changes and not isinstance(changes[name], bool):
|
||||
raise ValueError(f"{name} must be true or false.")
|
||||
return BackupConfig.from_dict({**current.to_dict(), **changes})
|
||||
|
||||
|
||||
async def load_config() -> BackupConfig:
|
||||
raw = await get_admin_setting(CONFIG_SETTING, "")
|
||||
try:
|
||||
data = json.loads(raw) if raw else {}
|
||||
except ValueError:
|
||||
data = {}
|
||||
return BackupConfig.from_dict(data if isinstance(data, dict) else {})
|
||||
|
||||
|
||||
async def save_config(config: BackupConfig) -> None:
|
||||
await set_admin_setting(CONFIG_SETTING, json.dumps(config.to_dict(), sort_keys=True))
|
||||
|
||||
|
||||
async def load_secret() -> tuple[str | None, bool]:
|
||||
"""(the S3 secret or None, whether one is stored).
|
||||
|
||||
Stored but None means it was sealed under another SECRET_KEY — after a
|
||||
restore onto a different instance, say — and has to be entered again.
|
||||
"""
|
||||
stored = await get_admin_setting(SECRET_SETTING, "")
|
||||
return (unseal(stored), bool(stored))
|
||||
|
||||
|
||||
async def save_secret(secret: str) -> None:
|
||||
await set_admin_setting(SECRET_SETTING, seal(secret) if secret else "")
|
||||
|
||||
|
||||
def s3_target(config: BackupConfig, secret: str) -> S3Target:
|
||||
return S3Target(
|
||||
endpoint=config.s3_endpoint, bucket=config.s3_bucket,
|
||||
access_key=config.s3_access_key, secret_key=secret,
|
||||
region=config.s3_region or "us-east-1", verify_tls=config.s3_verify_tls,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Delivered:
|
||||
name: str
|
||||
size_bytes: int
|
||||
where: str
|
||||
|
||||
|
||||
async def deliver(
|
||||
payload: dict, config: BackupConfig, secret: str | None, at: datetime,
|
||||
*, app_version: str, transport=None,
|
||||
) -> Delivered:
|
||||
"""Write `payload` as the dated backup file and put it at the destination.
|
||||
|
||||
The file is written off the event loop: serialising a full export is
|
||||
seconds of CPU on a large corpus, and the app keeps serving meanwhile.
|
||||
"""
|
||||
problems = config.problems(has_secret=bool(secret))
|
||||
if problems:
|
||||
raise DestinationError(" ".join(problems))
|
||||
|
||||
if config.destination == "folder":
|
||||
folder = Path(config.folder_path)
|
||||
if not folder.is_dir():
|
||||
raise DestinationError(
|
||||
f"{folder} isn't there. Mount the backup share at that path, "
|
||||
"or choose the folder it is mounted at."
|
||||
)
|
||||
try:
|
||||
path = await asyncio.to_thread(
|
||||
backup_file.write_archive, payload, folder, at, app_version=app_version,
|
||||
)
|
||||
except PermissionError:
|
||||
raise DestinationError(f"Grimoire isn't allowed to write in {folder}.") from None
|
||||
return Delivered(path.name, path.stat().st_size, str(folder))
|
||||
|
||||
target = s3_target(config, secret or "")
|
||||
with tempfile.TemporaryDirectory(prefix="grimoire-backup-") as staging:
|
||||
path = await asyncio.to_thread(
|
||||
backup_file.write_archive, payload, Path(staging), at, app_version=app_version,
|
||||
)
|
||||
size = path.stat().st_size
|
||||
try:
|
||||
await put_file(target, path.name, path, transport=transport)
|
||||
except S3Error as exc:
|
||||
raise DestinationError(str(exc)) from None
|
||||
return Delivered(path.name, size, target.url())
|
||||
|
||||
|
||||
async def check_destination(
|
||||
config: BackupConfig, secret: str | None, *, transport=None,
|
||||
) -> str:
|
||||
"""Ask the destination whether it would take a backup, without sending
|
||||
one. Returns what was found, in words; raises DestinationError otherwise.
|
||||
|
||||
The folder check writes and removes a dot-file, which a folder watcher
|
||||
ignores. The S3 check only lists: an append-only key could add a test
|
||||
object and then never remove it.
|
||||
"""
|
||||
problems = config.problems(has_secret=bool(secret))
|
||||
if problems:
|
||||
raise DestinationError(" ".join(problems))
|
||||
|
||||
if config.destination == "folder":
|
||||
folder = Path(config.folder_path)
|
||||
if not folder.is_dir():
|
||||
raise DestinationError(f"{folder} isn't there, or isn't a folder.")
|
||||
probe = folder / ".grimoire-write-check"
|
||||
try:
|
||||
await asyncio.to_thread(probe.write_bytes, b"")
|
||||
await asyncio.to_thread(probe.unlink)
|
||||
except PermissionError:
|
||||
raise DestinationError(f"Grimoire isn't allowed to write in {folder}.") from None
|
||||
except OSError as exc:
|
||||
raise DestinationError(f"Writing in {folder} failed: {exc.strerror or exc}.") from None
|
||||
count = await asyncio.to_thread(_count_backups, folder)
|
||||
return f"{folder} is writable and holds {count} Grimoire backup(s)."
|
||||
|
||||
target = s3_target(config, secret or "")
|
||||
try:
|
||||
objects = [
|
||||
obj
|
||||
for prefix in backup_file.NAME_PREFIXES
|
||||
for obj in await list_objects(target, prefix, transport=transport)
|
||||
]
|
||||
except S3Error as exc:
|
||||
raise DestinationError(str(exc)) from None
|
||||
return f"Bucket {config.s3_bucket} answered and holds {len(objects)} Grimoire backup(s)."
|
||||
|
||||
|
||||
def _count_backups(folder: Path) -> int:
|
||||
return sum(1 for name in os.listdir(folder) if backup_file.is_backup_name(name))
|
||||
@@ -0,0 +1,225 @@
|
||||
"""The full backup as a file on disk: one uncompressed tar per run.
|
||||
|
||||
The scheduled backup lands somewhere that keeps a history of it — a folder a
|
||||
backup service watches, an S3 bucket — and from there it is usually copied
|
||||
off-site by a deduplicating tool such as restic. Everything about the layout
|
||||
serves that copy: restic cuts a file into chunks at boundaries chosen by the
|
||||
content, so a chunk it has seen before costs nothing to store again. The file
|
||||
is therefore laid out so that what did not change between two runs is the
|
||||
same bytes, in the same place, both times.
|
||||
|
||||
- **Never compressed.** Compression rewrites every byte after the first
|
||||
change, so no chunk would ever match. restic compresses its own repository
|
||||
after deduplicating.
|
||||
- **One member per payload section**, `<section>.jsonl`, one row per line.
|
||||
Rows arrive in primary-key order (export_full_backup orders every query)
|
||||
and each is written with sorted keys and no spacing, so an unchanged row is
|
||||
identical bytes, an edit changes one line, and rows appended to a growing
|
||||
table land at the end of its member. A busy table never shifts the bytes of
|
||||
a quiet one.
|
||||
- **The parts that change every run live in `manifest.json` only**: the
|
||||
export time, the app version and the row counts. The member headers are
|
||||
fixed (mtime 0, owner 0, mode 0644) and the members come in one order: the
|
||||
manifest, then the sections by name.
|
||||
- **One file per run, under a new dated name**, so a service that keeps one
|
||||
file per backup can thin the history on its own schedule.
|
||||
|
||||
The payload inside is exactly `export_full_backup()`'s, so the coverage
|
||||
guards on that payload (tests/test_services_backup.py) cover this file too,
|
||||
and `read_archive` hands `restore_full_backup` the same dict it always took.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import IO
|
||||
|
||||
ARCHIVE_FORMAT = "grimoire-backup-tar"
|
||||
ARCHIVE_FORMAT_VERSION = 1
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
SECTION_SUFFIX = ".jsonl"
|
||||
NAME_PREFIX = "grimoire-"
|
||||
NAME_SUFFIX = ".tar"
|
||||
|
||||
# Grimoire was called Scribe, and archives it wrote carry that name in their
|
||||
# manifest and file name. They are read and counted like any other, so a
|
||||
# backup taken before the rename still restores.
|
||||
READABLE_FORMATS = (ARCHIVE_FORMAT, "scribe-backup-tar")
|
||||
NAME_PREFIXES = (NAME_PREFIX, "scribe-")
|
||||
|
||||
|
||||
def is_backup_name(name: str) -> bool:
|
||||
"""A file or object name this app's scheduled backup wrote."""
|
||||
return name.endswith(NAME_SUFFIX) and name.startswith(NAME_PREFIXES)
|
||||
|
||||
|
||||
def archive_name(at: datetime) -> str:
|
||||
"""`grimoire-20261015T030000Z.tar` — UTC, sortable, one per run."""
|
||||
return f"{NAME_PREFIX}{at.astimezone(timezone.utc):%Y%m%dT%H%M%SZ}{NAME_SUFFIX}"
|
||||
|
||||
|
||||
def _encode(value) -> str:
|
||||
# default=str matches the admin download's json.dumps, so a datetime that
|
||||
# reaches the payload unconverted reads back the same way on restore.
|
||||
return json.dumps(
|
||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str,
|
||||
)
|
||||
|
||||
|
||||
def _split(payload: dict) -> tuple[dict, dict[str, list]]:
|
||||
"""Separate the row sections from the header (version, scope, notices).
|
||||
|
||||
A section is a list of rows under a plain name. `_not_included` is a list
|
||||
too, but of table names: it is a notice, so it stays in the header.
|
||||
"""
|
||||
header: dict = {}
|
||||
sections: dict[str, list] = {}
|
||||
for key, value in payload.items():
|
||||
if isinstance(value, list) and not key.startswith("_"):
|
||||
sections[key] = value
|
||||
else:
|
||||
header[key] = value
|
||||
return header, sections
|
||||
|
||||
|
||||
def _member(name: str, data: bytes) -> tuple[tarfile.TarInfo, io.BytesIO]:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
info.mtime = 0
|
||||
info.mode = 0o644
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = ""
|
||||
return info, io.BytesIO(data)
|
||||
|
||||
|
||||
def write_archive_to(payload: dict, out: IO[bytes], *, app_version: str) -> dict:
|
||||
"""Write `payload` as the backup tar to an open binary file.
|
||||
|
||||
Returns the manifest, so the caller can report row counts without
|
||||
reading the file back.
|
||||
"""
|
||||
header, sections = _split(payload)
|
||||
names = sorted(sections)
|
||||
manifest = {
|
||||
"format": ARCHIVE_FORMAT,
|
||||
"format_version": ARCHIVE_FORMAT_VERSION,
|
||||
"app_version": app_version,
|
||||
"header": header,
|
||||
"sections": {name: len(sections[name]) for name in names},
|
||||
}
|
||||
with tarfile.open(fileobj=out, mode="w", format=tarfile.USTAR_FORMAT) as tar:
|
||||
tar.addfile(*_member(
|
||||
MANIFEST_NAME, (json.dumps(manifest, sort_keys=True, indent=2,
|
||||
default=str) + "\n").encode(),
|
||||
))
|
||||
for name in names:
|
||||
data = b"".join(
|
||||
_encode(row).encode() + b"\n" for row in sections[name]
|
||||
)
|
||||
tar.addfile(*_member(name + SECTION_SUFFIX, data))
|
||||
return manifest
|
||||
|
||||
|
||||
def write_archive(
|
||||
payload: dict, directory: Path, at: datetime, *, app_version: str,
|
||||
) -> Path:
|
||||
"""Write the backup tar into `directory` under its dated name, atomically.
|
||||
|
||||
The bytes go to a dot-prefixed `.part` file first, are fsync'd, and only
|
||||
then renamed to the final name. A watcher of the folder (or anything else
|
||||
listing it) never sees half a backup under a backup's name, and a run that
|
||||
fails leaves nothing behind but the error.
|
||||
"""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
final = directory / archive_name(at)
|
||||
part = directory / f".{final.name}.part"
|
||||
try:
|
||||
with open(part, "wb") as f:
|
||||
write_archive_to(payload, f, app_version=app_version)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(part, final)
|
||||
except BaseException:
|
||||
part.unlink(missing_ok=True)
|
||||
raise
|
||||
_fsync_dir(directory)
|
||||
return final
|
||||
|
||||
|
||||
def _fsync_dir(directory: Path) -> None:
|
||||
"""Make the rename itself durable. Some filesystems (and NFS mounts)
|
||||
refuse a directory fsync; the file's own fsync already happened, so that
|
||||
refusal is not a failure of the backup."""
|
||||
try:
|
||||
fd = os.open(directory, os.O_RDONLY)
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
os.fsync(fd)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
class ArchiveError(ValueError):
|
||||
"""The file is not a backup tar this version can read, or is damaged."""
|
||||
|
||||
|
||||
def is_archive(fileobj: IO[bytes]) -> bool:
|
||||
"""True when `fileobj` (seekable, at its start) holds a tar. Leaves the
|
||||
position where it was, so the caller can fall back to reading JSON."""
|
||||
pos = fileobj.tell()
|
||||
try:
|
||||
head = fileobj.read(tarfile.BLOCKSIZE)
|
||||
finally:
|
||||
fileobj.seek(pos)
|
||||
return len(head) == tarfile.BLOCKSIZE and head[257:262] == b"ustar"
|
||||
|
||||
|
||||
def read_archive(fileobj: IO[bytes]) -> dict:
|
||||
"""Read a backup tar back into the payload `restore_full_backup` takes.
|
||||
|
||||
Every section the manifest names must be present with the row count it
|
||||
names: a member that went missing or was cut short is an error here, not
|
||||
a restore that reports success with less than it was given.
|
||||
"""
|
||||
try:
|
||||
tar = tarfile.open(fileobj=fileobj, mode="r:")
|
||||
except tarfile.TarError as exc:
|
||||
raise ArchiveError(f"not a backup archive: {exc}") from exc
|
||||
with tar:
|
||||
manifest = json.loads(_read_member(tar, MANIFEST_NAME))
|
||||
if manifest.get("format") not in READABLE_FORMATS:
|
||||
raise ArchiveError("not a Grimoire backup archive (no Grimoire manifest)")
|
||||
if manifest.get("format_version", 0) > ARCHIVE_FORMAT_VERSION:
|
||||
raise ArchiveError(
|
||||
f"archive format {manifest['format_version']} is newer than "
|
||||
f"this Grimoire reads ({ARCHIVE_FORMAT_VERSION}); upgrade first"
|
||||
)
|
||||
payload = dict(manifest.get("header") or {})
|
||||
for name, count in (manifest.get("sections") or {}).items():
|
||||
raw = _read_member(tar, name + SECTION_SUFFIX)
|
||||
rows = [json.loads(line) for line in raw.splitlines() if line]
|
||||
if len(rows) != count:
|
||||
raise ArchiveError(
|
||||
f"{name}{SECTION_SUFFIX} holds {len(rows)} rows; "
|
||||
f"the manifest says {count}"
|
||||
)
|
||||
payload[name] = rows
|
||||
return payload
|
||||
|
||||
|
||||
def _read_member(tar: tarfile.TarFile, name: str) -> bytes:
|
||||
try:
|
||||
member = tar.getmember(name)
|
||||
except KeyError as exc:
|
||||
raise ArchiveError(f"the archive has no {name}") from exc
|
||||
f = tar.extractfile(member)
|
||||
if f is None:
|
||||
raise ArchiveError(f"{name} in the archive is not a file")
|
||||
return f.read()
|
||||
@@ -0,0 +1,188 @@
|
||||
"""The scheduled backup: when it runs, running it, and what each run did.
|
||||
|
||||
ONE APScheduler job (services/scheduler.py) fires at the top of every hour and
|
||||
reads the config at fire time: when backups are on and the hour is the chosen
|
||||
one, it runs. Reading at fire time rather than scheduling the chosen hour
|
||||
keeps the app's startup free of a database read (a lifespan hook that waits
|
||||
on the database is a worker that never serves), and a changed hour
|
||||
needs no reschedule.
|
||||
|
||||
A run exports the full backup, writes it as the dated tar
|
||||
(services/backup_file.py) and delivers it (services/backup_destination.py).
|
||||
Its outcome is kept as one entry in a bounded list — the newest
|
||||
`KEEP_RUNS` — in the `backup_runs` admin setting, like DB maintenance keeps
|
||||
its last run. A failure is logged as an error AND kept with its reason in
|
||||
words, so the Settings page can say what went wrong without anyone reading
|
||||
logs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from grimoire.services.backup_destination import (
|
||||
DestinationError, deliver, load_config, load_secret,
|
||||
)
|
||||
from grimoire.services.scheduler import ScheduledJob
|
||||
from grimoire.services.settings import get_admin_setting, set_admin_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RUNS_SETTING = "backup_runs"
|
||||
KEEP_RUNS = 30
|
||||
# A daily backup that hasn't succeeded for this long is overdue: a day plus
|
||||
# room for one slow run, so a backup that lands at 02:40 instead of 02:05
|
||||
# does not flicker the page into a warning.
|
||||
STALE_AFTER = timedelta(hours=26)
|
||||
# The whole run's deadline, because a wait with no deadline is a bug. The
|
||||
# S3 upload has its own 15-minute timeout, but a folder on a hung NFS mount
|
||||
# has none, and a run that never returns holds the lock — every later backup
|
||||
# would be skipped as "already running", silently, forever.
|
||||
RUN_DEADLINE = timedelta(minutes=60)
|
||||
|
||||
_LOCK = asyncio.Lock()
|
||||
_TASKS: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
class BackupBusy(Exception):
|
||||
"""A backup is already running."""
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
return _LOCK.locked()
|
||||
|
||||
|
||||
async def list_runs() -> list[dict]:
|
||||
"""The kept runs, newest first."""
|
||||
raw = await get_admin_setting(RUNS_SETTING, "")
|
||||
try:
|
||||
runs = json.loads(raw) if raw else []
|
||||
except ValueError:
|
||||
return []
|
||||
return [r for r in runs if isinstance(r, dict)] if isinstance(runs, list) else []
|
||||
|
||||
|
||||
async def _record(run: dict) -> None:
|
||||
runs = [run, *await list_runs()][:KEEP_RUNS]
|
||||
await set_admin_setting(RUNS_SETTING, json.dumps(runs))
|
||||
|
||||
|
||||
def summarise(runs: list[dict], *, enabled: bool, now: datetime) -> dict:
|
||||
"""What the page leads with: when the last good backup was, and whether
|
||||
backups are overdue. Pure, so the stale rule is tested on its own."""
|
||||
last_ok = next((r for r in runs if r.get("status") == "ok"), None)
|
||||
last_ok_at = _parse(last_ok.get("finished_at")) if last_ok else None
|
||||
if not enabled:
|
||||
overdue = False
|
||||
elif last_ok_at is None:
|
||||
# On, and never succeeded: overdue once the first scheduled chance
|
||||
# has passed. Before any run at all, there is nothing to be late for.
|
||||
overdue = bool(runs)
|
||||
else:
|
||||
overdue = now - last_ok_at > STALE_AFTER
|
||||
return {
|
||||
"last_run": runs[0] if runs else None,
|
||||
"last_success": last_ok,
|
||||
"overdue": overdue,
|
||||
}
|
||||
|
||||
|
||||
def _parse(value) -> datetime | None:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def run_backup(trigger: str) -> dict:
|
||||
"""Run one backup now and keep its outcome. Raises BackupBusy when one is
|
||||
already running; never raises for a failed backup — that is the outcome.
|
||||
"""
|
||||
if _LOCK.locked():
|
||||
raise BackupBusy()
|
||||
async with _LOCK:
|
||||
started = datetime.now(timezone.utc)
|
||||
clock = time.monotonic()
|
||||
run: dict = {"started_at": started.isoformat(), "trigger": trigger}
|
||||
try:
|
||||
config = await load_config()
|
||||
secret, _ = await load_secret()
|
||||
run["destination"] = config.destination
|
||||
delivered = await asyncio.wait_for(
|
||||
_export_and_deliver(config, secret, started), RUN_DEADLINE.total_seconds(),
|
||||
)
|
||||
run.update(status="ok", file=delivered.name,
|
||||
size_bytes=delivered.size_bytes, where=delivered.where)
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
run.update(status="failed", error=(
|
||||
f"The backup didn't finish within {int(RUN_DEADLINE.total_seconds() // 60)} "
|
||||
"minutes and was abandoned. The destination may be hanging."
|
||||
))
|
||||
except DestinationError as exc:
|
||||
run.update(status="failed", error=str(exc))
|
||||
except Exception as exc:
|
||||
logger.exception("backup run failed")
|
||||
run.update(status="failed", error=f"The backup failed: {exc}")
|
||||
run["finished_at"] = datetime.now(timezone.utc).isoformat()
|
||||
run["duration_s"] = round(time.monotonic() - clock, 1)
|
||||
if run["status"] == "ok":
|
||||
logger.info("backup: wrote %s (%d bytes) to %s in %.1fs",
|
||||
run["file"], run["size_bytes"], run["where"], run["duration_s"])
|
||||
else:
|
||||
logger.error("backup: %s", run["error"])
|
||||
try:
|
||||
await _record(run)
|
||||
except Exception:
|
||||
# The backup itself is done (or failed) either way; losing its
|
||||
# entry in the history must not turn into an unhandled error in a
|
||||
# scheduler thread. The log line above still says what happened.
|
||||
logger.exception("backup: could not keep this run in the history")
|
||||
return run
|
||||
|
||||
|
||||
async def _export_and_deliver(config, secret, started):
|
||||
from grimoire.services.backup import export_full_backup
|
||||
payload = await export_full_backup()
|
||||
return await deliver(
|
||||
payload, config, secret, started,
|
||||
app_version=os.environ.get("APP_VERSION", "dev"),
|
||||
)
|
||||
|
||||
|
||||
def start_backup_now() -> None:
|
||||
"""Start a manual backup in the background (the request returns at once;
|
||||
the page reads progress from `is_running` and the run list). Raises
|
||||
BackupBusy rather than queueing a second run behind the first."""
|
||||
if _LOCK.locked():
|
||||
raise BackupBusy()
|
||||
task = asyncio.get_running_loop().create_task(run_backup("manual"))
|
||||
_TASKS.add(task)
|
||||
task.add_done_callback(_TASKS.discard)
|
||||
|
||||
|
||||
async def _hourly_check() -> None:
|
||||
config = await load_config()
|
||||
if not config.enabled or datetime.now(timezone.utc).hour != config.hour:
|
||||
return
|
||||
try:
|
||||
await run_backup("scheduled")
|
||||
except BackupBusy:
|
||||
logger.info("backup: a run is already going; the scheduled one is skipped")
|
||||
|
||||
|
||||
_JOB = ScheduledJob("backup_hourly_check", _hourly_check, label="Backup")
|
||||
|
||||
|
||||
def start_backup_scheduler(loop: asyncio.AbstractEventLoop) -> None:
|
||||
_JOB.start(loop, CronTrigger(minute=0, timezone="UTC"),
|
||||
describe="hourly; runs at the configured UTC hour")
|
||||
|
||||
|
||||
def stop_backup_scheduler() -> None:
|
||||
_JOB.stop()
|
||||
@@ -0,0 +1,199 @@
|
||||
"""A small S3 client for sending backups: AWS Signature Version 4 over httpx,
|
||||
path-style addressing (endpoint/bucket/key), and the two calls a backup needs.
|
||||
|
||||
The signing is checked against AWS's published SigV4 examples in
|
||||
tests/test_services_backup_s3.py.
|
||||
|
||||
Grimoire puts and lists; it never deletes. The bucket keeps the history and
|
||||
thins it on its own policy, so a key that may only add objects is enough —
|
||||
and is the safer key to hand a service, because a stolen one cannot wipe the
|
||||
history it writes into.
|
||||
|
||||
Written by hand rather than with boto3: two calls don't justify a large
|
||||
dependency, and signing is a pure function tests can hold to AWS's own
|
||||
published examples.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import hmac
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
|
||||
TIMEOUT_SECONDS = 30.0
|
||||
# A backup is tens of megabytes; a slow link still gets this long to send one.
|
||||
UPLOAD_TIMEOUT_SECONDS = 15 * 60
|
||||
CHUNK = 1 << 20
|
||||
|
||||
|
||||
class S3Error(Exception):
|
||||
"""Why a call to the bucket failed, said to the person."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class S3Target:
|
||||
endpoint: str # http(s)://host[:port][/prefix], no bucket
|
||||
bucket: str
|
||||
access_key: str
|
||||
secret_key: str
|
||||
region: str = "us-east-1"
|
||||
verify_tls: bool = True
|
||||
|
||||
def url(self, key: str = "") -> str:
|
||||
base = f"{self.endpoint.rstrip('/')}/{self.bucket}"
|
||||
return f"{base}/{key}" if key else f"{base}/"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: str
|
||||
|
||||
|
||||
# ── Signing ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _encode(text: str, keep_slash: bool = False) -> str:
|
||||
return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
|
||||
|
||||
|
||||
def _hmac(key: bytes, msg: str) -> bytes:
|
||||
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||
|
||||
|
||||
def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
|
||||
access_key: str, secret_key: str, region: str, at: datetime,
|
||||
service: str = "s3") -> dict[str, str]:
|
||||
"""The request's headers with x-amz-date, x-amz-content-sha256 and
|
||||
Authorization added. Every header given is signed, Host always."""
|
||||
u = httpx.URL(url)
|
||||
stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
day = stamp[:8]
|
||||
out = dict(headers)
|
||||
out["x-amz-date"] = stamp
|
||||
out["x-amz-content-sha256"] = payload_sha256
|
||||
signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
|
||||
signed["host"] = u.netloc.decode()
|
||||
names = sorted(signed)
|
||||
|
||||
query = sorted((_encode(k), _encode(v)) for k, v in u.params.multi_items())
|
||||
canonical = "\n".join([
|
||||
method.upper(),
|
||||
_encode(u.path or "/", keep_slash=True),
|
||||
"&".join(f"{k}={v}" for k, v in query),
|
||||
"".join(f"{n}:{signed[n]}\n" for n in names),
|
||||
";".join(names),
|
||||
payload_sha256,
|
||||
])
|
||||
scope = f"{day}/{region}/{service}/aws4_request"
|
||||
to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
|
||||
hashlib.sha256(canonical.encode()).hexdigest()])
|
||||
key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
|
||||
service), "aws4_request")
|
||||
signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
|
||||
f"SignedHeaders={';'.join(names)}, Signature={signature}")
|
||||
return out
|
||||
|
||||
|
||||
# ── Calls ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _says(response: httpx.Response) -> str:
|
||||
"""An S3 error response in words: its Code and Message when it has them."""
|
||||
try:
|
||||
root = ET.fromstring(response.content)
|
||||
code = root.findtext("Code") or ""
|
||||
message = root.findtext("Message") or ""
|
||||
except ET.ParseError:
|
||||
code, message = "", ""
|
||||
hints = {
|
||||
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||
"InvalidAccessKeyId": "the access key isn't known there",
|
||||
"NoSuchBucket": "there is no such bucket",
|
||||
"AccessDenied": "the key isn't allowed to do that",
|
||||
}
|
||||
said = hints.get(code) or message or response.reason_phrase
|
||||
return f"HTTP {response.status_code}{f' {code}' if code else ''}: {said}."
|
||||
|
||||
|
||||
def _reach_error(target: S3Target, exc: httpx.HTTPError) -> S3Error:
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
return S3Error(f"{target.endpoint} didn't answer in time.")
|
||||
return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
async def put_file(target: S3Target, key: str, path: Path, *,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
now: datetime | None = None) -> None:
|
||||
"""Upload one file as one object, streamed from disk."""
|
||||
digest = await asyncio.to_thread(file_sha256, path)
|
||||
size = path.stat().st_size
|
||||
url = target.url(key)
|
||||
headers = sign("PUT", url, {"Content-Length": str(size),
|
||||
"Content-Type": "application/x-tar"},
|
||||
digest, target.access_key, target.secret_key, target.region,
|
||||
now or datetime.now(timezone.utc))
|
||||
|
||||
async def body():
|
||||
with open(path, "rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
yield chunk
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(verify=target.verify_tls,
|
||||
timeout=UPLOAD_TIMEOUT_SECONDS,
|
||||
transport=transport) as client:
|
||||
response = await client.put(url, headers=headers, content=body())
|
||||
except httpx.HTTPError as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
if response.status_code // 100 != 2:
|
||||
raise S3Error(f"Upload of {key} refused: {_says(response)}")
|
||||
|
||||
|
||||
async def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
now: datetime | None = None) -> list[StoredObject]:
|
||||
"""Up to `limit` objects under `prefix` (ListObjectsV2, one page)."""
|
||||
params = {"list-type": "2", "max-keys": str(limit)}
|
||||
if prefix:
|
||||
params["prefix"] = prefix
|
||||
url = str(httpx.URL(target.url(), params=params))
|
||||
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
|
||||
target.region, now or datetime.now(timezone.utc))
|
||||
try:
|
||||
async with httpx.AsyncClient(verify=target.verify_tls, timeout=TIMEOUT_SECONDS,
|
||||
transport=transport) as client:
|
||||
response = await client.get(url, headers=headers)
|
||||
except httpx.HTTPError as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
if response.status_code != 200:
|
||||
raise S3Error(f"Listing {target.bucket} refused: {_says(response)}")
|
||||
try:
|
||||
root = ET.fromstring(response.content)
|
||||
except ET.ParseError:
|
||||
raise S3Error(
|
||||
f"{target.endpoint} answered a listing with something that isn't S3's XML."
|
||||
) from None
|
||||
out = []
|
||||
for item in root.iter():
|
||||
if item.tag.rsplit("}", 1)[-1] != "Contents":
|
||||
continue
|
||||
fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
|
||||
out.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
|
||||
fields.get("LastModified", "")))
|
||||
return out
|
||||
@@ -0,0 +1,287 @@
|
||||
"""The global canonical area vocabulary, and the mapping from a project's
|
||||
Systems onto it.
|
||||
|
||||
A `System` is per-project. Nothing outside a project can reference one, so a
|
||||
rule that spans projects has no way to say "this is about CI" without chaining
|
||||
itself to one project's row. `CanonicalSystem` is that join key, and it is
|
||||
GLOBAL — no `user_id`, so a shared project inherits the vocabulary instead of
|
||||
re-earning it.
|
||||
|
||||
Two rules govern everything here:
|
||||
|
||||
- **Associate, never rewrite.** Mapping a System sets `systems.canonical_id`
|
||||
and nothing else. The local name stays whatever the project calls the area,
|
||||
and `record_systems` is never touched — no record's tags move.
|
||||
- **Propose, never decide.** An exact slug hit is mechanical and maps on
|
||||
request; anything short of that is a PROPOSAL a human confirms. "CI &
|
||||
Release" vs "CI & runners" is a judgment call, and the cost of guessing it
|
||||
wrong silently is a rule surfacing in the wrong project.
|
||||
|
||||
Reads are open to any authenticated caller (the catalog is shared vocabulary,
|
||||
not user data). Writes to the catalog itself are admin-only: a global table
|
||||
that anyone can extend is how a shared vocabulary stops being shared.
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.canonical_system import CanonicalSystem
|
||||
from grimoire.models.system import System
|
||||
from grimoire.services import access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tokens that carry no meaning for matching — "&" becomes "and" before the
|
||||
# split, so it would otherwise dominate the overlap score of every pair.
|
||||
_NOISE_TOKENS = frozenset({"and", "the", "a", "of"})
|
||||
|
||||
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def canonical_slug(name: str) -> str:
|
||||
"""The match key for an area name — NOT a display value.
|
||||
|
||||
Folds exactly the spelling differences that produced three names for one
|
||||
area on the author's instance: `CI & Release`, `CI and Release` and
|
||||
`CI & release` all slug to `ci-and-release`, so they map mechanically.
|
||||
A real difference survives: `CI & runners` slugs to `ci-and-runners` and
|
||||
goes through the proposal path where a human decides.
|
||||
"""
|
||||
lowered = name.strip().lower().replace("&", " and ")
|
||||
return "-".join(_NON_ALNUM.sub(" ", lowered).split())
|
||||
|
||||
|
||||
def _tokens(slug: str) -> frozenset[str]:
|
||||
return frozenset(slug.split("-")) - _NOISE_TOKENS
|
||||
|
||||
|
||||
async def _is_admin(user_id: int) -> bool:
|
||||
return await access.is_instance_admin(user_id)
|
||||
|
||||
|
||||
async def list_canonical_systems() -> list[CanonicalSystem]:
|
||||
"""The whole catalog, in display order. Global — no ownership filter."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(CanonicalSystem)
|
||||
.where(CanonicalSystem.deleted_at.is_(None))
|
||||
.order_by(CanonicalSystem.order_index.asc(), CanonicalSystem.name.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_canonical_system(canonical_id: int) -> CanonicalSystem | None:
|
||||
async with async_session() as session:
|
||||
entry = await session.get(CanonicalSystem, canonical_id)
|
||||
return entry if entry is not None and entry.deleted_at is None else None
|
||||
|
||||
|
||||
async def find_by_name(name: str) -> CanonicalSystem | None:
|
||||
"""The exact-slug lookup — the mechanical half of matching."""
|
||||
slug = canonical_slug(name)
|
||||
if not slug:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
return await session.scalar(
|
||||
select(CanonicalSystem).where(
|
||||
CanonicalSystem.slug == slug,
|
||||
CanonicalSystem.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _overlap(local: frozenset[str], other: frozenset[str]) -> float:
|
||||
return len(local & other) / max(len(local | other), 1)
|
||||
|
||||
|
||||
async def best_overlap(name: str, catalog: list | None = None) -> dict | None:
|
||||
"""The closest catalog entry that shares a meaningful word, or None.
|
||||
|
||||
The ONE scorer behind both offers: the create-time suggestion and the
|
||||
review surface. Two scorers would eventually disagree about which area a
|
||||
name resembles, and the operator would be asked one question at create
|
||||
time and a different one at review.
|
||||
|
||||
The threshold is any shared meaningful word, deliberately generous: a
|
||||
wrong offer costs one dismissal, a missing one costs a mapping nobody
|
||||
thinks to make again. Nothing here ever applies — `overlap` is always an
|
||||
offer (see propose_mappings).
|
||||
"""
|
||||
slug = canonical_slug(name)
|
||||
if not slug:
|
||||
return None
|
||||
local = _tokens(slug)
|
||||
if not local:
|
||||
return None
|
||||
# A caller already holding the catalog passes it: this runs once per
|
||||
# unmapped System in the review sweep, and re-reading the table each time
|
||||
# would make an N+1 out of a report.
|
||||
if catalog is None:
|
||||
catalog = await list_canonical_systems()
|
||||
best, best_score = None, 0.0
|
||||
for entry in catalog:
|
||||
score = _overlap(local, _tokens(entry.slug))
|
||||
if score > best_score:
|
||||
best, best_score = entry, score
|
||||
if best is None or best_score <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": best.id, "name": best.name,
|
||||
"basis": "overlap", "score": round(best_score, 3),
|
||||
}
|
||||
|
||||
|
||||
async def create_canonical_system(
|
||||
user_id: int, name: str, description: str | None = None,
|
||||
) -> CanonicalSystem | dict | None:
|
||||
"""Add an area to the global catalog. Admin only.
|
||||
|
||||
Duplicate-gated on the SLUG, not the raw name, so "CI and Release" cannot
|
||||
be added alongside "CI & Release" — that is the drift this table exists to
|
||||
end. Returns the existing entry's id instead of creating a second one.
|
||||
"""
|
||||
if not await _is_admin(user_id):
|
||||
return None
|
||||
slug = canonical_slug(name)
|
||||
if not slug:
|
||||
return None
|
||||
existing = await find_by_name(name)
|
||||
if existing is not None:
|
||||
return {
|
||||
"duplicate": True,
|
||||
"existing_id": existing.id,
|
||||
"message": (
|
||||
f"'{existing.name}' (#{existing.id}) already covers this area — "
|
||||
f"both names reduce to '{slug}'. Map Systems to it, or "
|
||||
"update_canonical_system if the charter needs revising."
|
||||
),
|
||||
}
|
||||
async with async_session() as session:
|
||||
highest = await session.scalar(
|
||||
select(CanonicalSystem.order_index)
|
||||
.order_by(CanonicalSystem.order_index.desc())
|
||||
.limit(1)
|
||||
)
|
||||
entry = CanonicalSystem(
|
||||
name=" ".join(name.split()),
|
||||
slug=slug,
|
||||
description=description,
|
||||
order_index=(highest or 0) + 1,
|
||||
)
|
||||
session.add(entry)
|
||||
await session.commit()
|
||||
await session.refresh(entry)
|
||||
return entry
|
||||
|
||||
|
||||
async def update_canonical_system(
|
||||
user_id: int, canonical_id: int, **fields: object,
|
||||
) -> CanonicalSystem | None:
|
||||
"""Rename or re-charter a catalog entry. Admin only.
|
||||
|
||||
A rename recomputes the slug — the display name and the match key must not
|
||||
be allowed to disagree, or the exact-match path silently stops finding it.
|
||||
"""
|
||||
if not await _is_admin(user_id):
|
||||
return None
|
||||
allowed = {"name", "description", "order_index"}
|
||||
async with async_session() as session:
|
||||
entry = await session.get(CanonicalSystem, canonical_id)
|
||||
if entry is None or entry.deleted_at is not None:
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if key in allowed and value is not None:
|
||||
setattr(entry, key, value)
|
||||
if "name" in fields and fields["name"]:
|
||||
entry.name = " ".join(str(fields["name"]).split())
|
||||
entry.slug = canonical_slug(entry.name)
|
||||
entry.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(entry)
|
||||
return entry
|
||||
|
||||
|
||||
async def set_system_canonical(
|
||||
user_id: int, system_id: int, canonical_id: int | None,
|
||||
) -> System | None:
|
||||
"""Map (or unmap) one project System onto a catalog entry.
|
||||
|
||||
Authorised by the PROJECT, not the catalog: mapping changes the project's
|
||||
row, so project write access is the right gate — never a bare owner
|
||||
filter. Passing None clears the mapping.
|
||||
|
||||
Touches `canonical_id` and nothing else — the System's own name, charter
|
||||
and record associations are left exactly as they are.
|
||||
"""
|
||||
if canonical_id is not None and await get_canonical_system(canonical_id) is None:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
system = await session.get(System, system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
if not await access.can_write_project(user_id, system.project_id):
|
||||
return None
|
||||
system.canonical_id = canonical_id
|
||||
system.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(system)
|
||||
return system
|
||||
|
||||
|
||||
async def propose_mappings(user_id: int, project_id: int) -> list[dict]:
|
||||
"""Suggest a catalog entry for each of a project's UNMAPPED Systems.
|
||||
|
||||
Returns proposals, never applied changes — `set_system_canonical` is the
|
||||
only thing that writes. Each carries a `basis` so the reviewer knows what
|
||||
they are approving:
|
||||
|
||||
- `exact` — the two names reduce to the same slug. Mechanical.
|
||||
- `overlap` — they share a meaningful word ("CI & runners" / "CI &
|
||||
Release"). A judgment call, and the reason this is a proposal at all.
|
||||
|
||||
A System with no plausible match simply gets no proposal: unmapped is a
|
||||
perfectly good resting state, so silence here is an answer, not a gap.
|
||||
"""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return []
|
||||
catalog = await list_canonical_systems()
|
||||
if not catalog:
|
||||
return []
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(System).where(
|
||||
System.project_id == project_id,
|
||||
System.canonical_id.is_(None),
|
||||
System.deleted_at.is_(None),
|
||||
).order_by(System.order_index.asc(), System.created_at.asc())
|
||||
)
|
||||
systems = list(result.scalars().all())
|
||||
|
||||
by_slug = {entry.slug: entry for entry in catalog}
|
||||
proposals: list[dict] = []
|
||||
for system in systems:
|
||||
slug = canonical_slug(system.name)
|
||||
if not slug:
|
||||
continue
|
||||
exact = by_slug.get(slug)
|
||||
if exact is not None:
|
||||
match = {"id": exact.id, "name": exact.name, "basis": "exact", "score": 1.0}
|
||||
else:
|
||||
# Same scorer the create-time offer uses, so the two surfaces can
|
||||
# never name different areas for one System.
|
||||
match = await best_overlap(system.name, catalog)
|
||||
if match is None:
|
||||
continue
|
||||
proposals.append({
|
||||
"system_id": system.id,
|
||||
"system_name": system.name,
|
||||
"canonical_id": match["id"],
|
||||
"canonical_name": match["name"],
|
||||
"basis": match["basis"],
|
||||
"score": match["score"],
|
||||
})
|
||||
proposals.sort(key=lambda p: (-p["score"], p["system_name"]))
|
||||
return proposals
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,197 @@
|
||||
"""Dashboard aggregation — assembles the /dashboard landing payload.
|
||||
|
||||
One call: most-recently-active projects (each -> active milestones -> open
|
||||
tasks), recently-completed tasks, week stats. Owner-scoped,
|
||||
trashed rows excluded. Each section is independent — a failure returns its
|
||||
empty value rather than blanking the page.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.milestone import Milestone
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.services import milestones as milestones_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
N_PROJECTS = 3 # most-recently-active projects shown
|
||||
TASKS_PER_GROUP = 5 # open-task cap per milestone / no-milestone group
|
||||
RECENT_DONE_LIMIT = 8 # recently-completed tasks shown
|
||||
OPEN_ISSUES_LIMIT = 10 # open issues shown on the dashboard
|
||||
WINDOW_DAYS = 7 # look-back window (done items, week stats)
|
||||
_OPEN = ["todo", "in_progress"]
|
||||
|
||||
|
||||
def _open_order():
|
||||
"""in-progress first -> priority high..none -> most-recently-updated."""
|
||||
status_rank = case((Note.status == "in_progress", 0), else_=1)
|
||||
priority_rank = case(
|
||||
(Note.priority == "high", 0), (Note.priority == "medium", 1),
|
||||
(Note.priority == "low", 2), else_=3,
|
||||
)
|
||||
return status_rank, priority_rank, Note.updated_at.desc()
|
||||
|
||||
|
||||
def _task_row(n: Note) -> dict:
|
||||
# task_kind rides along so a dashboard row can show WHAT KIND of work it
|
||||
# is, not just how it is going. Omitting it made the kind badge render
|
||||
# nothing here while working everywhere else — the badge was correct and
|
||||
# the payload was short, which reads as "no issues in this list" rather
|
||||
# than as a missing field.
|
||||
return {"id": n.id, "title": n.title, "status": n.status,
|
||||
"priority": n.priority or "none", "task_kind": n.task_kind}
|
||||
|
||||
|
||||
async def _safe(coro, empty):
|
||||
try:
|
||||
return await coro
|
||||
except Exception:
|
||||
logger.warning("dashboard section failed", exc_info=True)
|
||||
return empty
|
||||
|
||||
|
||||
async def _open_issues(user_id: int) -> list[dict]:
|
||||
"""Open issues (task_kind='issue', not done/cancelled) across the owner's
|
||||
projects, ranked like the other task lists. Owner-scoped, matching the rest
|
||||
of the dashboard."""
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(
|
||||
Note.id, Note.title, Note.status, Note.priority,
|
||||
Note.project_id, Project.title,
|
||||
)
|
||||
.join(Project, Project.id == Note.project_id, isouter=True)
|
||||
.where(
|
||||
Note.user_id == user_id,
|
||||
Note.task_kind == "issue",
|
||||
Note.status.in_(_OPEN),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
.order_by(*_open_order())
|
||||
.limit(OPEN_ISSUES_LIMIT)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": iid, "title": title, "status": status,
|
||||
"priority": priority or "none",
|
||||
"project_id": pid, "project_title": pname,
|
||||
}
|
||||
for iid, title, status, priority, pid, pname in rows.fetchall()
|
||||
]
|
||||
|
||||
|
||||
async def build_dashboard(user_id: int) -> dict:
|
||||
return {
|
||||
"active_projects": await _safe(_active_projects(user_id), []),
|
||||
"recently_completed": await _safe(_recently_completed(user_id), []),
|
||||
"open_issues": await _safe(_open_issues(user_id), []),
|
||||
"week_stats": await _safe(_week_stats(user_id), {}),
|
||||
}
|
||||
|
||||
|
||||
async def _active_projects(user_id: int) -> list[dict]:
|
||||
so, po, ro = _open_order()
|
||||
async with async_session() as session:
|
||||
recency = (
|
||||
select(Note.project_id, func.max(Note.updated_at).label("last"))
|
||||
.where(Note.user_id == user_id, Note.deleted_at.is_(None),
|
||||
Note.project_id.isnot(None))
|
||||
.group_by(Note.project_id).subquery()
|
||||
)
|
||||
prows = (await session.execute(
|
||||
select(Project, recency.c.last)
|
||||
.outerjoin(recency, Project.id == recency.c.project_id)
|
||||
.where(Project.user_id == user_id, Project.status == "active",
|
||||
Project.deleted_at.is_(None))
|
||||
.order_by(func.coalesce(recency.c.last, Project.updated_at).desc())
|
||||
.limit(N_PROJECTS)
|
||||
)).all()
|
||||
|
||||
out = []
|
||||
for project, last in prows:
|
||||
counts = (await session.execute(
|
||||
select(
|
||||
func.count(Note.id).filter(Note.status.in_(_OPEN)),
|
||||
func.count(Note.id).filter(Note.status == "done"),
|
||||
func.count(Note.id).filter(Note.status.in_(_OPEN + ["done"])),
|
||||
).where(Note.user_id == user_id, Note.project_id == project.id,
|
||||
Note.deleted_at.is_(None))
|
||||
)).one()
|
||||
open_count, done_count, resolved_total = counts
|
||||
progress_pct = round(done_count / resolved_total * 100, 1) if resolved_total else 0.0
|
||||
|
||||
mrows = (await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.user_id == user_id, Milestone.project_id == project.id,
|
||||
Milestone.status == "active", Milestone.deleted_at.is_(None))
|
||||
.order_by(Milestone.order_index.asc())
|
||||
)).scalars().all()
|
||||
milestones = []
|
||||
for m in mrows:
|
||||
tasks = (await session.execute(
|
||||
select(Note).where(
|
||||
Note.user_id == user_id, Note.milestone_id == m.id,
|
||||
Note.status.in_(_OPEN), Note.deleted_at.is_(None))
|
||||
.order_by(so, po, ro).limit(TASKS_PER_GROUP)
|
||||
)).scalars().all()
|
||||
if not tasks:
|
||||
continue
|
||||
prog = await milestones_svc.get_milestone_progress(m.id)
|
||||
milestones.append({
|
||||
"id": m.id, "title": m.title, "progress_pct": prog["pct"],
|
||||
"open_tasks": [_task_row(t) for t in tasks],
|
||||
})
|
||||
|
||||
no_ms = (await session.execute(
|
||||
select(Note).where(
|
||||
Note.user_id == user_id, Note.project_id == project.id,
|
||||
Note.milestone_id.is_(None), Note.status.in_(_OPEN),
|
||||
Note.deleted_at.is_(None))
|
||||
.order_by(so, po, ro).limit(TASKS_PER_GROUP)
|
||||
)).scalars().all()
|
||||
|
||||
out.append({
|
||||
"id": project.id, "title": project.title, "color": project.color,
|
||||
"last_activity": (last or project.updated_at).isoformat(),
|
||||
"open_count": open_count, "done_count": done_count,
|
||||
"progress_pct": progress_pct,
|
||||
"milestones": milestones,
|
||||
"no_milestone": [_task_row(t) for t in no_ms],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def _recently_completed(user_id: int) -> list[dict]:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=WINDOW_DAYS)
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(Note, Project.title)
|
||||
.outerjoin(Project, Note.project_id == Project.id)
|
||||
.where(Note.user_id == user_id, Note.status == "done",
|
||||
Note.deleted_at.is_(None), Note.completed_at.isnot(None),
|
||||
Note.completed_at >= cutoff)
|
||||
.order_by(Note.completed_at.desc()).limit(RECENT_DONE_LIMIT)
|
||||
)).all()
|
||||
return [{"id": n.id, "title": n.title, "project_title": ptitle,
|
||||
"completed_at": iso(n.completed_at)} for n, ptitle in rows]
|
||||
|
||||
|
||||
async def _week_stats(user_id: int) -> dict:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=WINDOW_DAYS)
|
||||
async with async_session() as session:
|
||||
row = (await session.execute(
|
||||
select(
|
||||
func.count(Note.id).filter(Note.status == "done", Note.completed_at >= cutoff),
|
||||
func.count(Note.id).filter(Note.status.in_(_OPEN)),
|
||||
func.count(Note.id).filter(Note.status == "in_progress"),
|
||||
func.count(Note.id).filter(Note.task_kind == "plan", Note.status.in_(_OPEN)),
|
||||
).where(Note.user_id == user_id, Note.deleted_at.is_(None))
|
||||
)).one()
|
||||
return {"completed_this_week": row[0], "open_total": row[1],
|
||||
"in_progress": row[2], "active_plans": row[3]}
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Basic Postgres maintenance — targeted VACUUM (ANALYZE).
|
||||
|
||||
Postgres autovacuum already reclaims dead tuples and refreshes planner stats
|
||||
in the background. This adds a daily, off-hours top-up over the handful of
|
||||
tables the retention/purge sweeps churn hardest (log retention, notification
|
||||
purge, auth-token purge, trash purge, version pruning) — so the bloat those
|
||||
bulk DELETEs leave behind is collected promptly and the planner keeps good
|
||||
stats. It is deliberately narrow; the rest of the schema is left to autovacuum.
|
||||
|
||||
Security: table names cannot be parameterized in SQL, so VACUUM is only ever
|
||||
issued against names from the hardcoded MAINTENANCE_TABLES allowlist — there is
|
||||
no path from user input to the table name.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from grimoire.models import async_session, engine
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.services.settings import get_admin_setting, set_admin_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# High-churn tables — exactly the ones the scheduled delete sweeps hit:
|
||||
# app_logs ← log retention (hourly)
|
||||
# notifications ← read-notification purge (hourly)
|
||||
# password_reset_tokens ← auth-token purge (daily)
|
||||
# invitation_tokens ← auth-token purge (daily)
|
||||
# notes ← trash purge (daily) + general churn
|
||||
# note_versions ← version-pinning prune (daily)
|
||||
MAINTENANCE_TABLES: tuple[str, ...] = (
|
||||
"app_logs",
|
||||
"notifications",
|
||||
"password_reset_tokens",
|
||||
"invitation_tokens",
|
||||
"notes",
|
||||
"note_versions",
|
||||
)
|
||||
|
||||
_LAST_RUN_KEY = "db_maintenance_last_run"
|
||||
|
||||
|
||||
async def run_maintenance(tables: tuple[str, ...] | list[str] | None = None) -> dict:
|
||||
"""Run VACUUM (ANALYZE) over the allowlisted high-churn tables.
|
||||
|
||||
VACUUM cannot run inside a transaction block, so this uses a dedicated
|
||||
AUTOCOMMIT connection rather than the usual session. Each table is
|
||||
vacuumed independently; one failure is logged and does not abort the rest.
|
||||
|
||||
Returns a summary dict (also persisted as the db_maintenance_last_run
|
||||
admin setting) of shape:
|
||||
{"started_at": iso, "elapsed_ms": int, "tables": [
|
||||
{"table": str, "ok": bool, "elapsed_ms": int, "error": str|None}
|
||||
]}
|
||||
"""
|
||||
# Only ever operate on the closed allowlist — never an arbitrary name.
|
||||
requested = tuple(tables) if tables is not None else MAINTENANCE_TABLES
|
||||
targets = [t for t in requested if t in MAINTENANCE_TABLES]
|
||||
|
||||
started = time.monotonic()
|
||||
started_at = datetime.now(timezone.utc).isoformat()
|
||||
results: list[dict] = []
|
||||
|
||||
async with engine.connect() as conn:
|
||||
# On AsyncConnection, execution_options() is a coroutine and MUST be
|
||||
# awaited (it returns the connection with AUTOCOMMIT set — VACUUM can't
|
||||
# run inside a transaction block).
|
||||
autocommit = await conn.execution_options(isolation_level="AUTOCOMMIT")
|
||||
for table in targets:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
# table is from the allowlist above — safe to interpolate.
|
||||
await autocommit.exec_driver_sql(f"VACUUM (ANALYZE) {table}")
|
||||
results.append({
|
||||
"table": table,
|
||||
"ok": True,
|
||||
"elapsed_ms": int((time.monotonic() - t0) * 1000),
|
||||
"error": None,
|
||||
})
|
||||
except Exception as exc: # noqa: BLE001 — record per-table, keep going
|
||||
logger.exception("VACUUM (ANALYZE) failed for %s", table)
|
||||
results.append({
|
||||
"table": table,
|
||||
"ok": False,
|
||||
"elapsed_ms": int((time.monotonic() - t0) * 1000),
|
||||
"error": str(exc),
|
||||
})
|
||||
|
||||
summary = {
|
||||
"started_at": started_at,
|
||||
"elapsed_ms": int((time.monotonic() - started) * 1000),
|
||||
"tables": results,
|
||||
}
|
||||
ok = sum(1 for r in results if r["ok"])
|
||||
logger.info(
|
||||
"DB maintenance: vacuumed %d/%d table(s) in %dms",
|
||||
ok, len(results), summary["elapsed_ms"],
|
||||
)
|
||||
try:
|
||||
await set_admin_setting(_LAST_RUN_KEY, json.dumps(summary))
|
||||
except Exception: # noqa: BLE001 — persisting the summary is best-effort
|
||||
logger.warning("Failed to persist db_maintenance last-run summary", exc_info=True)
|
||||
return summary
|
||||
|
||||
|
||||
# Read-only table-health query against Postgres' own statistics views. The key
|
||||
# signal is the dead-tuple ratio (dead / (live + dead)) — that IS bloat; a high
|
||||
# ratio on a large table means autovacuum isn't keeping up. Timestamps confirm
|
||||
# maintenance is actually running on each table.
|
||||
_HEALTH_SQL = text("""
|
||||
SELECT
|
||||
relname AS table_name,
|
||||
n_live_tup AS live,
|
||||
n_dead_tup AS dead,
|
||||
CASE WHEN n_live_tup + n_dead_tup > 0
|
||||
THEN round(100.0 * n_dead_tup / (n_live_tup + n_dead_tup), 1)
|
||||
ELSE 0 END AS dead_pct,
|
||||
pg_total_relation_size(relid) AS total_bytes,
|
||||
n_mod_since_analyze AS mod_since_analyze,
|
||||
GREATEST(last_vacuum, last_autovacuum) AS last_vacuum,
|
||||
GREATEST(last_analyze, last_autoanalyze) AS last_analyze
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY n_dead_tup DESC, total_bytes DESC
|
||||
""")
|
||||
|
||||
|
||||
async def get_table_health() -> dict:
|
||||
"""Per-table health from Postgres statistics + the total database size.
|
||||
|
||||
Read-only (system catalogs only). Returns:
|
||||
{"db_bytes": int, "tables": [
|
||||
{"table": str, "live": int, "dead": int, "dead_pct": float,
|
||||
"total_bytes": int, "mod_since_analyze": int,
|
||||
"last_vacuum": iso|None, "last_analyze": iso|None}
|
||||
]}
|
||||
"""
|
||||
async with async_session() as session:
|
||||
db_bytes = (
|
||||
await session.execute(text("SELECT pg_database_size(current_database())"))
|
||||
).scalar() or 0
|
||||
rows = (await session.execute(_HEALTH_SQL)).mappings().all()
|
||||
|
||||
tables = [
|
||||
{
|
||||
"table": r["table_name"],
|
||||
"live": int(r["live"] or 0),
|
||||
"dead": int(r["dead"] or 0),
|
||||
"dead_pct": float(r["dead_pct"] or 0),
|
||||
"total_bytes": int(r["total_bytes"] or 0),
|
||||
"mod_since_analyze": int(r["mod_since_analyze"] or 0),
|
||||
"last_vacuum": iso(r["last_vacuum"]),
|
||||
"last_analyze": iso(r["last_analyze"]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
return {"db_bytes": int(db_bytes), "tables": tables}
|
||||
|
||||
|
||||
async def get_last_run() -> dict | None:
|
||||
"""Return the most recent run summary, or None if it has never run."""
|
||||
raw = await get_admin_setting(_LAST_RUN_KEY, "")
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Daily APScheduler cron for basic DB maintenance (targeted VACUUM ANALYZE).
|
||||
|
||||
One ScheduledJob (services/scheduler.py). Scheduled for 04:00 UTC by default
|
||||
— after the 03:30 trash purge — so it collects the dead tuples that night's
|
||||
delete sweeps leave behind.
|
||||
|
||||
Two things are operator-tunable from the admin Settings card:
|
||||
- db_maintenance_enabled ("true"/"false") — checked at fire time, so toggling
|
||||
it needs no reschedule.
|
||||
- db_maintenance_hour ("0".."23", UTC) — the cron hour. Changing it calls
|
||||
reschedule_db_maintenance() so the live job moves without a restart.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from grimoire.services.scheduler import ScheduledJob
|
||||
from grimoire.services.settings import get_admin_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JOB_ID = "db_maintenance_vacuum"
|
||||
_DEFAULT_HOUR = 4
|
||||
|
||||
# HOW LONG A COLD START WILL WAIT FOR THIS ONE SETTING.
|
||||
#
|
||||
# This read is the FIRST database call in the app's `before_serving` hook, and
|
||||
# a lifespan hook that does not return is a worker that never serves. If the
|
||||
# app starts while the database host is stalled on storage, a query with no
|
||||
# deadline hangs, Hypercorn kills the worker at its 60-second lifespan timeout,
|
||||
# and nothing retries a failed lifespan — a few minutes of disk trouble becomes
|
||||
# an outage that only a human restart clears.
|
||||
#
|
||||
# Three seconds because the honest requirement is "don't hold up the boot",
|
||||
# not "get the right hour". The value is one small indexed row on a local
|
||||
# database: under any healthy condition this returns in single-digit
|
||||
# milliseconds, so the timeout can only ever fire when something is already
|
||||
# badly wrong — which is exactly the moment the app must come up anyway.
|
||||
_STARTUP_READ_TIMEOUT = 3.0
|
||||
|
||||
|
||||
async def get_maintenance_hour() -> int:
|
||||
"""The configured run-hour (UTC, 0–23), clamped; default 04:00.
|
||||
|
||||
BOUNDED, because the caller is a lifespan hook. The fallback is the same
|
||||
one this function gives a value it cannot parse: a database that will not
|
||||
answer in three seconds is the same class of "no usable value here". The
|
||||
failure is a logged line and a default hour rather than the application
|
||||
failing to start.
|
||||
|
||||
Degrading to the default is the right trade in both directions: the cost of
|
||||
being wrong is that a VACUUM runs at 04:00 instead of the configured hour,
|
||||
for one boot, on an instance whose disk is in trouble. The cost of waiting
|
||||
is the whole instance.
|
||||
"""
|
||||
try:
|
||||
raw = await asyncio.wait_for(
|
||||
get_admin_setting("db_maintenance_hour", str(_DEFAULT_HOUR)),
|
||||
timeout=_STARTUP_READ_TIMEOUT,
|
||||
)
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
# WARNING, not debug: this is never normal, and it is the breadcrumb
|
||||
# that names a stalled database in seconds instead of requiring
|
||||
# Postgres's own log to be read.
|
||||
logger.warning(
|
||||
"db maintenance: reading db_maintenance_hour exceeded %.1fs; "
|
||||
"starting with the default %02d:00 UTC. The database is slow or "
|
||||
"unreachable — this is a symptom, not the disease.",
|
||||
_STARTUP_READ_TIMEOUT, _DEFAULT_HOUR,
|
||||
)
|
||||
return _DEFAULT_HOUR
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"db maintenance: could not read db_maintenance_hour; starting "
|
||||
"with the default %02d:00 UTC", _DEFAULT_HOUR, exc_info=True,
|
||||
)
|
||||
return _DEFAULT_HOUR
|
||||
try:
|
||||
hour = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_HOUR
|
||||
return hour if 0 <= hour <= 23 else _DEFAULT_HOUR
|
||||
|
||||
|
||||
async def is_maintenance_enabled() -> bool:
|
||||
"""Whether the scheduled run is enabled (default on)."""
|
||||
return (await get_admin_setting("db_maintenance_enabled", "true")) != "false"
|
||||
|
||||
|
||||
async def _run_maintenance() -> None:
|
||||
if not await is_maintenance_enabled():
|
||||
logger.debug("db maintenance: disabled, skipping scheduled run")
|
||||
return
|
||||
from grimoire.services.db_maintenance import run_maintenance
|
||||
await run_maintenance()
|
||||
|
||||
|
||||
_JOB = ScheduledJob(_JOB_ID, _run_maintenance, label="DB maintenance")
|
||||
|
||||
|
||||
def _trigger(hour: int) -> CronTrigger:
|
||||
hour = hour if 0 <= hour <= 23 else _DEFAULT_HOUR
|
||||
return CronTrigger(hour=hour, minute=0, timezone="UTC")
|
||||
|
||||
|
||||
def start_db_maintenance_scheduler(
|
||||
loop: asyncio.AbstractEventLoop, hour: int = _DEFAULT_HOUR
|
||||
) -> None:
|
||||
"""Start the daily job. `hour` is the configured UTC run-hour, resolved by
|
||||
the caller (which has an async context) via get_maintenance_hour() — passed
|
||||
in rather than read here so we never block the event loop at startup. The
|
||||
job's enabled-gate is re-checked at every fire, so only the hour is needed
|
||||
up front."""
|
||||
hour = hour if 0 <= hour <= 23 else _DEFAULT_HOUR
|
||||
_JOB.start(loop, _trigger(hour), describe=f"daily {hour:02d}:00 UTC")
|
||||
|
||||
|
||||
def reschedule_db_maintenance(hour: int) -> None:
|
||||
"""Move the live job to a new UTC hour (called when the admin changes it)."""
|
||||
hour = hour if 0 <= hour <= 23 else _DEFAULT_HOUR
|
||||
_JOB.reschedule(_trigger(hour), describe=f"{hour:02d}:00 UTC")
|
||||
|
||||
|
||||
def stop_db_maintenance_scheduler() -> None:
|
||||
_JOB.stop()
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Wait for the database before the app starts.
|
||||
|
||||
The container runs migrations first, and the database may still be starting
|
||||
(or its host may not resolve yet) when it does. Waiting here, with a deadline,
|
||||
lets the app start on an orchestrator's defaults instead of relying on a
|
||||
healthcheck or a restart policy to retry it.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
import asyncpg
|
||||
from sqlalchemy.engine import URL, make_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Long enough for a first boot that initialises the data directory on slow
|
||||
# storage; past it the process exits non-zero and the orchestrator restarts it.
|
||||
DEFAULT_DEADLINE_S = 300.0
|
||||
|
||||
# Errors that mean "not up yet". Anything else (a wrong password, a missing
|
||||
# database) fails at once, because waiting would not change the answer.
|
||||
_TRANSIENT = (
|
||||
OSError, # refused, unreachable, DNS not resolving yet, connect timeout
|
||||
asyncpg.CannotConnectNowError, # "the database system is starting up"
|
||||
asyncpg.TooManyConnectionsError,
|
||||
)
|
||||
|
||||
|
||||
def asyncpg_dsn(database_url: str | URL) -> str:
|
||||
"""The SQLAlchemy URL as a plain DSN asyncpg accepts."""
|
||||
url = make_url(database_url).set(drivername="postgresql")
|
||||
return url.render_as_string(hide_password=False)
|
||||
|
||||
|
||||
async def wait_for_database(
|
||||
database_url: str | URL,
|
||||
deadline_s: float = DEFAULT_DEADLINE_S,
|
||||
*,
|
||||
connect: Callable[..., Awaitable] = asyncpg.connect,
|
||||
sleep: Callable[[float], Awaitable] = asyncio.sleep,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
"""Return once a connection succeeds; raise the last error after the deadline.
|
||||
|
||||
Retries are spaced 1, 2, 4 … seconds apart, capped at 10, so a database
|
||||
that is down for minutes sees a handful of attempts rather than a flood.
|
||||
"""
|
||||
dsn = asyncpg_dsn(database_url)
|
||||
started = clock()
|
||||
delay = 1.0
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
try:
|
||||
conn = await connect(dsn, timeout=10)
|
||||
except _TRANSIENT as exc:
|
||||
waited = clock() - started
|
||||
if waited + delay > deadline_s:
|
||||
logger.error(
|
||||
"Database still unreachable after %.0fs (%d attempts): %s",
|
||||
waited, attempt, exc,
|
||||
)
|
||||
raise
|
||||
logger.warning(
|
||||
"Waiting for the database (attempt %d, %.0fs so far): %s",
|
||||
attempt, waited, exc,
|
||||
)
|
||||
await sleep(delay)
|
||||
delay = min(delay * 2, 10.0)
|
||||
continue
|
||||
await conn.close()
|
||||
if attempt > 1:
|
||||
logger.info("Database reachable after %d attempts", attempt)
|
||||
return
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,251 @@
|
||||
"""The design-system cascade, as pure functions over already-loaded rows.
|
||||
|
||||
Deliberately free of every database and service import — including
|
||||
`services/access.py`, which needs `ancestry` to answer "can this caller read
|
||||
this system?" and would otherwise form an import cycle with the service that
|
||||
needs `access` back. A module that imports nothing can be imported by both.
|
||||
|
||||
Being pure is also what makes the cascade rule testable without a database:
|
||||
these take a plain `{id: parent_id}` map, so a test states the shape of a
|
||||
hierarchy in one literal instead of building one.
|
||||
"""
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
# The mode key that applies when no more specific one does. Emitted CSS puts it
|
||||
# on the base selector and every other key on a mode selector, mirroring how a
|
||||
# stylesheet is actually written: light on `:root`, dark layered over it.
|
||||
BASE_MODE = "base"
|
||||
|
||||
|
||||
def ancestry(system_id: int, parents: Mapping[int, int | None]) -> list[int]:
|
||||
"""The inheritance chain from `system_id` up to its root, nearest first.
|
||||
|
||||
Includes `system_id` itself at index 0, because resolution wants
|
||||
deepest-to-shallowest and the system being resolved is the deepest link.
|
||||
|
||||
A system missing from `parents` terminates the chain rather than raising: a
|
||||
parent whose row was soft-deleted or filtered out is a truncated chain, not
|
||||
a failed request.
|
||||
|
||||
The visited-set is defensive, not the primary guard — writes already refuse
|
||||
to create a cycle (`would_cycle`). It is here because a loop introduced by a
|
||||
direct DB edit or a future bug must degrade to a truncated chain instead of
|
||||
spinning forever. Truncation shows up in the result; a hang shows up as an
|
||||
outage.
|
||||
"""
|
||||
chain: list[int] = []
|
||||
seen: set[int] = set()
|
||||
current: int | None = system_id
|
||||
while current is not None and current not in seen:
|
||||
seen.add(current)
|
||||
chain.append(current)
|
||||
current = parents.get(current)
|
||||
return chain
|
||||
|
||||
|
||||
def would_cycle(
|
||||
system_id: int,
|
||||
proposed_parent_id: int | None,
|
||||
parents: Mapping[int, int | None],
|
||||
) -> bool:
|
||||
"""Would making `proposed_parent_id` the parent of `system_id` close a loop?
|
||||
|
||||
True when the proposed parent IS the system, or already inherits from it.
|
||||
|
||||
The walk goes UP from the proposed parent, which is the cheap direction —
|
||||
each system has at most one parent, so the chain is a line. Asking the
|
||||
equivalent downward question ("is the proposed parent among my
|
||||
descendants?") would mean searching a whole forest for the same answer.
|
||||
|
||||
`proposed_parent_id=None` clears the parent and can never cycle.
|
||||
"""
|
||||
if proposed_parent_id is None:
|
||||
return False
|
||||
if proposed_parent_id == system_id:
|
||||
return True
|
||||
return system_id in ancestry(proposed_parent_id, parents)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolution — flattening a chain into an effective token set
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Contribution:
|
||||
"""One system's offer for one token in one mode."""
|
||||
system_id: int
|
||||
value: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedToken:
|
||||
"""A token after the cascade, carrying the whole argument rather than the verdict.
|
||||
|
||||
`contributions` holds every system that supplied a value, per mode, DEEPEST
|
||||
FIRST — so `[0]` is the winner and `[1:]` are what it shadowed. Storing the
|
||||
contest rather than a winner plus a separate provenance field means there is
|
||||
nothing to keep in sync: "which system supplied this?" and "what did it
|
||||
override?" are both reads of the same list, and they cannot disagree.
|
||||
|
||||
Provenance is per MODE, not per token, because overriding is. A system that
|
||||
deepens one accent for light backgrounds while leaving dark alone owns the
|
||||
base value and inherits the dark one, and a token-level "overridden here"
|
||||
flag would have to lie about one of them.
|
||||
"""
|
||||
name: str
|
||||
contributions: dict[str, tuple[Contribution, ...]]
|
||||
group_name: str | None
|
||||
purpose: str | None
|
||||
rationale: str | None
|
||||
supersedes: tuple[str, ...]
|
||||
order_index: int
|
||||
|
||||
@property
|
||||
def value_by_mode(self) -> dict[str, str]:
|
||||
"""The effective value for each mode — the winner of each contest."""
|
||||
return {mode: entries[0].value for mode, entries in self.contributions.items()}
|
||||
|
||||
@property
|
||||
def origin_by_mode(self) -> dict[str, int]:
|
||||
"""Which system supplied each mode's effective value."""
|
||||
return {mode: entries[0].system_id for mode, entries in self.contributions.items()}
|
||||
|
||||
def value_for(self, mode: str) -> str | None:
|
||||
"""The value to render in `mode`, falling back to the base mode.
|
||||
|
||||
This is the read rule the storage shape implies: a token that is not
|
||||
mode-dependent carries only `base`, and asking it for "dark" must yield
|
||||
the base value rather than nothing.
|
||||
"""
|
||||
entries = self.contributions.get(mode) or self.contributions.get(BASE_MODE)
|
||||
return entries[0].value if entries else None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Payload shape for both surfaces — and it carries the SHADOWED entries.
|
||||
|
||||
Serialising only the winner would throw away the provenance at the last
|
||||
step, which is the one thing this type exists to preserve. `contributions`
|
||||
is the audit trail; `value_by_mode` / `origin_by_mode` are alongside it so
|
||||
a client renders without re-deriving anything, and cannot derive it
|
||||
differently.
|
||||
"""
|
||||
return {
|
||||
"name": self.name,
|
||||
"group_name": self.group_name,
|
||||
"purpose": self.purpose,
|
||||
"rationale": self.rationale,
|
||||
"supersedes": list(self.supersedes),
|
||||
"order_index": self.order_index,
|
||||
"value_by_mode": self.value_by_mode,
|
||||
"origin_by_mode": self.origin_by_mode,
|
||||
"contributions": {
|
||||
mode: [
|
||||
{"system_id": c.system_id, "value": c.value} for c in entries
|
||||
]
|
||||
for mode, entries in self.contributions.items()
|
||||
},
|
||||
}
|
||||
|
||||
def is_overridden_in(self, system_id: int) -> bool:
|
||||
"""Does `system_id` win any mode of this token AND shadow something?
|
||||
|
||||
The distinction the UI needs: a token this system introduced is not an
|
||||
override, and a token it merely inherits is not either.
|
||||
"""
|
||||
return any(
|
||||
len(entries) > 1 and entries[0].system_id == system_id
|
||||
for entries in self.contributions.values()
|
||||
)
|
||||
|
||||
|
||||
def _sort_key(token: ResolvedToken) -> tuple:
|
||||
# Ungrouped tokens sort last rather than first: a design system that has
|
||||
# started grouping should read as its groups, with the not-yet-filed
|
||||
# remainder at the end.
|
||||
return (token.group_name is None, token.group_name or "", token.order_index, token.name)
|
||||
|
||||
|
||||
def resolve_tokens(
|
||||
system_id: int,
|
||||
parents: Mapping[int, int | None],
|
||||
tokens_by_system: Mapping[int, Sequence],
|
||||
) -> list[ResolvedToken]:
|
||||
"""Flatten a system's inheritance chain into its effective token set.
|
||||
|
||||
Walks from `system_id` up to the root and applies tokens by name, deepest
|
||||
winning. That is the CSS cascade — precedence by name along a parent chain —
|
||||
rather than an analogy to it, which is why the storage model and the
|
||||
stylesheet model came out the same shape.
|
||||
|
||||
`tokens_by_system` maps a system id to its own token rows. Any object with
|
||||
`.name`, `.value_by_mode`, `.group_name`, `.purpose` and `.order_index` will
|
||||
do, so a test can state a hierarchy in literals and the service can pass ORM
|
||||
rows to the same function.
|
||||
|
||||
The result includes tokens the system never mentions — inheriting one is
|
||||
what puts it in the effective set. A system with no tokens of its own
|
||||
resolves to its parent's set entire, which is the correct answer for an app
|
||||
that has not departed from the family yet.
|
||||
|
||||
Merging is per (name, MODE): a child that supplies only a dark value
|
||||
overrides only dark and keeps inheriting base. Metadata (`group_name`,
|
||||
`purpose`, `order_index`) cascades separately by the same deepest-wins rule,
|
||||
since a child overriding a value routinely leaves the family's description
|
||||
of what the token is FOR untouched — and inheriting it beats blanking it.
|
||||
"""
|
||||
chain = ancestry(system_id, parents)
|
||||
|
||||
contributions: dict[str, dict[str, list[Contribution]]] = {}
|
||||
metadata: dict[str, dict[str, object]] = {}
|
||||
|
||||
# Deepest first, so the first contribution seen for a (name, mode) wins and
|
||||
# every later one is a shadowed ancestor appended behind it.
|
||||
for depth_system_id in chain:
|
||||
for token in tokens_by_system.get(depth_system_id) or ():
|
||||
per_mode = contributions.setdefault(token.name, {})
|
||||
for mode, value in (token.value_by_mode or {}).items():
|
||||
per_mode.setdefault(mode, []).append(
|
||||
Contribution(system_id=depth_system_id, value=value)
|
||||
)
|
||||
|
||||
meta = metadata.setdefault(
|
||||
token.name,
|
||||
{
|
||||
"group_name": None, "purpose": None, "rationale": None,
|
||||
"supersedes": None, "order_index": None,
|
||||
},
|
||||
)
|
||||
for field in ("group_name", "purpose", "rationale"):
|
||||
if meta[field] is None:
|
||||
meta[field] = getattr(token, field, None)
|
||||
# order_index alone treats 0 as UNSTATED rather than "first",
|
||||
# because 0 is the column default. Reading it as a real value would
|
||||
# let any child override drag its token to the top of the group and
|
||||
# lose the family's ordering — a visible reshuffle in return for a
|
||||
# change that only touched a colour.
|
||||
if not meta["order_index"]:
|
||||
meta["order_index"] = getattr(token, "order_index", 0) or None
|
||||
# `supersedes` cascades on EMPTINESS, not on None: a child that
|
||||
# overrides a colour and says nothing about which literals it
|
||||
# replaces should keep the family's declaration, and an empty list
|
||||
# is what "said nothing" looks like once the column is NOT NULL.
|
||||
# A child that states its own list replaces the whole thing.
|
||||
if not meta["supersedes"]:
|
||||
meta["supersedes"] = tuple(getattr(token, "supersedes", None) or ()) or None
|
||||
|
||||
resolved = [
|
||||
ResolvedToken(
|
||||
name=name,
|
||||
contributions={
|
||||
mode: tuple(entries) for mode, entries in per_mode.items()
|
||||
},
|
||||
group_name=metadata[name]["group_name"],
|
||||
purpose=metadata[name]["purpose"],
|
||||
rationale=metadata[name]["rationale"],
|
||||
supersedes=metadata[name]["supersedes"] or (),
|
||||
order_index=metadata[name]["order_index"] or 0,
|
||||
)
|
||||
for name, per_mode in contributions.items()
|
||||
]
|
||||
return sorted(resolved, key=_sort_key)
|
||||
@@ -0,0 +1,184 @@
|
||||
"""A starter set of token ROLES, offered when a design system is created.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
A literal gets written into a stylesheet when there is no role to reach for.
|
||||
A house style with no "text on a filled colour" role gets a pure-white literal
|
||||
at every call site that needs one — not out of defiance, but because nothing
|
||||
exists to write instead. The correction is not a better ban list. It is
|
||||
declaring the missing role.
|
||||
|
||||
So the useful moment is CREATION. A system whose roles are named on day one
|
||||
never presents the occasion for a literal, and never needs a list of values it
|
||||
forbids.
|
||||
|
||||
WHAT SHIPS AND WHAT DOES NOT
|
||||
----------------------------
|
||||
The ROLES ship: `surface-page`, `text-primary`, `action-destructive` are
|
||||
generic CSS-design vocabulary, not one operator's kit. Every install that has a
|
||||
page has a page background.
|
||||
|
||||
The VALUES never ship. Each token is created with an empty `value_by_mode`, so
|
||||
a fresh system is a set of named, deliberately-unanswered questions. No hex
|
||||
appears anywhere in this file, and none should ever be added to it — a default
|
||||
palette would be this operator's palette wearing product clothes.
|
||||
|
||||
A valueless token is already legible downstream: `render_stylesheet` emits it as
|
||||
a commented-out declaration in its group, and `stylesheet_for_system`
|
||||
reports it under `valueless`. So a blank role reads as "to be decided" rather
|
||||
than as breakage, without anything new.
|
||||
|
||||
THE PREFIX IS THE INSTALL'S
|
||||
---------------------------
|
||||
A house prefix such as `--fs-` is one family's convention, not the
|
||||
product's. The prefix is a parameter with a neutral default; a caller that has
|
||||
a house convention passes it. Baking one in would put one family's naming into
|
||||
every install.
|
||||
|
||||
FLAT, NOT PRESET
|
||||
----------------
|
||||
One list, every group individually skippable, all on by default. Presets
|
||||
keyed to app shape — web / CLI / docs — are not offered: they would require the product to hold opinions about
|
||||
app categories, and a wrong category is worse than a generic list someone
|
||||
prunes once.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
DEFAULT_TOKEN_PREFIX = "--ds-"
|
||||
|
||||
# group -> (what the group is for, ((role suffix, purpose), ...))
|
||||
#
|
||||
# Purposes are written as the QUESTION the operator is answering, because that
|
||||
# is what an unfilled role is. "Page background, the deepest surface" tells you
|
||||
# what to put there; "Colour 1" does not.
|
||||
STARTER_ROLE_GROUPS: dict[str, tuple[str, tuple[tuple[str, str], ...]]] = {
|
||||
"surface": (
|
||||
"Backgrounds, by elevation",
|
||||
(
|
||||
("surface-page", "Page background, the deepest surface"),
|
||||
("surface-raised", "Cards and raised elements"),
|
||||
("surface-hover", "Hovered surfaces, secondary elevation"),
|
||||
),
|
||||
),
|
||||
"text": (
|
||||
"Foreground colours, by emphasis",
|
||||
(
|
||||
("text-primary", "Primary text on a page or raised surface"),
|
||||
("text-secondary", "Secondary text and captions"),
|
||||
("text-tertiary", "Hints and metadata"),
|
||||
# The role whose absence caused 76 literals. It is in the starter
|
||||
# set deliberately: text on a filled colour is NOT the page text
|
||||
# colour, because the surface under it does not change with the
|
||||
# mode while the page does.
|
||||
("text-on-action", "Text on a filled colour — buttons, badges"),
|
||||
),
|
||||
),
|
||||
"action": (
|
||||
"What the user can do — kept separate from the accent, which is identity",
|
||||
(
|
||||
("action-primary", "The confirming action: Save, Submit"),
|
||||
("action-secondary", "Non-destructive alternates"),
|
||||
("action-destructive", "Irreversible actions — delete, revoke"),
|
||||
),
|
||||
),
|
||||
"semantic": (
|
||||
"What the system is telling you",
|
||||
(
|
||||
("success", "Something worked"),
|
||||
("warning", "Something needs attention"),
|
||||
("error", "Something failed — distinct from destructive"),
|
||||
("info", "Neutral information"),
|
||||
),
|
||||
),
|
||||
"border": (
|
||||
"Boundaries and dividers",
|
||||
(
|
||||
("border-color", "The line colour itself"),
|
||||
("border", "The default structural border, as a shorthand"),
|
||||
("border-hover", "Border on hover or emphasis"),
|
||||
("border-active", "Selected or current — the one border that may carry the accent"),
|
||||
),
|
||||
),
|
||||
"accent": (
|
||||
"This install's identity — not its actions",
|
||||
(
|
||||
("accent", "The single signature colour"),
|
||||
("accent-soft", "Tinted backgrounds — pills, tags"),
|
||||
("accent-faint", "The faintest wash"),
|
||||
),
|
||||
),
|
||||
"radius": (
|
||||
"Corner rounding",
|
||||
(
|
||||
("radius-sm", "Pills, tags, code spans"),
|
||||
("radius-md", "Buttons, inputs, small cards"),
|
||||
("radius-lg", "Cards, panels, modals"),
|
||||
),
|
||||
),
|
||||
"space": (
|
||||
"The spacing scale — a gap not on the scale is a decision to justify",
|
||||
tuple((f"space-{i}", f"Spacing step {i}") for i in range(1, 11)),
|
||||
),
|
||||
"motion": (
|
||||
"Transition timing — motion supports the interaction, never performs",
|
||||
(
|
||||
("ease", "The one easing curve, used by every transition"),
|
||||
("dur-fast", "Hovers, colour and border changes"),
|
||||
("dur-base", "Most state changes"),
|
||||
("dur-slow", "Larger surface or layout shifts"),
|
||||
),
|
||||
),
|
||||
"state": (
|
||||
"Cross-cutting states that are otherwise improvised per view",
|
||||
(
|
||||
("disabled-opacity", "Opacity for disabled controls"),
|
||||
("overlay", "Scrim behind modals and dialogs"),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
ALL_GROUPS: tuple[str, ...] = tuple(STARTER_ROLE_GROUPS)
|
||||
|
||||
|
||||
def starter_tokens(
|
||||
groups: list[str] | tuple[str, ...] | None = None,
|
||||
prefix: str = DEFAULT_TOKEN_PREFIX,
|
||||
) -> list[dict]:
|
||||
"""Token rows for the chosen groups — names and purposes only, no values.
|
||||
|
||||
`groups` of None means every group; an empty list means none, which is a
|
||||
real answer and not the same as None. An operator who wants three tokens
|
||||
should be able to get three.
|
||||
|
||||
Unknown group names are ignored rather than raising: this feeds a
|
||||
checkbox list, and a stale name from an older client should not fail a
|
||||
creation that is otherwise fine.
|
||||
"""
|
||||
chosen = ALL_GROUPS if groups is None else [g for g in groups if g in STARTER_ROLE_GROUPS]
|
||||
rows: list[dict] = []
|
||||
for group in chosen:
|
||||
_, roles = STARTER_ROLE_GROUPS[group]
|
||||
for index, (suffix, purpose) in enumerate(roles, start=1):
|
||||
rows.append({
|
||||
"name": f"{prefix}{suffix}",
|
||||
"group_name": group,
|
||||
"purpose": purpose,
|
||||
# Empty, not absent: the column is NOT NULL with a {} default,
|
||||
# so absence has exactly one spelling here as it does there.
|
||||
"value_by_mode": {},
|
||||
"order_index": index,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def describe_groups() -> list[dict]:
|
||||
"""The catalogue, for a UI to render as a checklist."""
|
||||
return [
|
||||
{
|
||||
"group": group,
|
||||
"description": description,
|
||||
"token_count": len(roles),
|
||||
"names": [suffix for suffix, _ in roles],
|
||||
}
|
||||
for group, (description, roles) in STARTER_ROLE_GROUPS.items()
|
||||
]
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Render a design system's resolved tokens as its master CSS sheet.
|
||||
|
||||
Pure, like `design_cascade` and for the same reasons: no database import, so the
|
||||
rendering rule is testable without one and the preview surface can reuse it.
|
||||
|
||||
WHAT THIS SHEET IS, AND DELIBERATELY IS NOT
|
||||
-------------------------------------------
|
||||
It declares **purpose tokens only** — custom properties, grouped by what they
|
||||
mean. It contains no rules for elements or classes: no `.btn-primary`, no
|
||||
`table`, no `input`.
|
||||
|
||||
That is the point rather than a limitation. A sheet that styled every element
|
||||
would restate the same handful of values once per element and grow with the UI;
|
||||
a sheet of purpose-named values states each once and is reused. Components —
|
||||
buttons, tables, input schemes — live as SNIPPETS that reference these names and
|
||||
carry prose about the idea, which is a surface that already exists and already
|
||||
has recall, locations, drift checks and merge.
|
||||
|
||||
So the division is: this sheet says what the values MEAN; snippets say what
|
||||
things LOOK LIKE, in terms of those values. A token named after an element
|
||||
(`--button-bg`) is the smell that the two have been mixed — it multiplies
|
||||
with every new element, where a purpose name (`--action-primary`) is reused.
|
||||
|
||||
SAFETY
|
||||
------
|
||||
Values are interpolated into a stylesheet, and design systems are shareable
|
||||
records. A value of `red; } body { display: none` in a system someone
|
||||
shared with you would otherwise inject CSS into your page. Everything rendered
|
||||
here is validated or dropped — never escaped-and-hoped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
|
||||
BASE_MODE = "base"
|
||||
|
||||
# A custom property name, strictly. Anything else is dropped rather than
|
||||
# sanitised: a name is an identifier, and a "cleaned up" identifier is a
|
||||
# different token than the one the operator recorded.
|
||||
_VALID_NAME = re.compile(r"^--[A-Za-z0-9_-]+$")
|
||||
|
||||
# Characters that would end the declaration, open a block, start an at-rule, or
|
||||
# begin a tag. A value containing any of them is not a value.
|
||||
_UNSAFE_VALUE = re.compile(r"[{};@<>]|\*/|/\*|\n|\r")
|
||||
|
||||
|
||||
def is_valid_token_name(name: str) -> bool:
|
||||
return bool(_VALID_NAME.match((name or "").strip()))
|
||||
|
||||
|
||||
def safe_value(value: str) -> str | None:
|
||||
"""A CSS value that cannot escape its declaration, or None.
|
||||
|
||||
Rejects rather than strips. A partially-sanitised value is a value the
|
||||
operator did not write, and silently rendering a different colour than the
|
||||
record holds is worse than rendering none — the sheet's whole claim is that
|
||||
it IS the record.
|
||||
"""
|
||||
candidate = (value or "").strip()
|
||||
if not candidate or _UNSAFE_VALUE.search(candidate):
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def safe_comment(text: str) -> str:
|
||||
"""Comment text that cannot close its comment or break the line."""
|
||||
return re.sub(r"\*/|/\*|[\n\r]", " ", (text or "")).strip()
|
||||
|
||||
|
||||
def selector_for_mode(mode: str, root_selector: str = ":root") -> str:
|
||||
"""Which selector a mode's declarations belong under.
|
||||
|
||||
`base` gets the caller's root selector; every other mode gets a bare
|
||||
attribute selector, matching the convention already in the codebase.
|
||||
|
||||
The root selector is a PARAMETER because a container-scoped preview cannot
|
||||
use `:root` — mode scoping is one-way (light lives on `:root`, dark layers
|
||||
over it), so a generator that hardcoded `:root` could
|
||||
not serve a preview at all.
|
||||
"""
|
||||
if mode == BASE_MODE:
|
||||
return root_selector
|
||||
safe_mode = re.sub(r"[^A-Za-z0-9_-]", "", mode)
|
||||
return f'[data-theme="{safe_mode}"]' if safe_mode else root_selector
|
||||
|
||||
|
||||
def _grouped(tokens: Sequence) -> list[tuple[str | None, list]]:
|
||||
"""Tokens by group, preserving the order they arrive in (already sorted)."""
|
||||
groups: dict[str | None, list] = {}
|
||||
for token in tokens:
|
||||
groups.setdefault(getattr(token, "group_name", None), []).append(token)
|
||||
return list(groups.items())
|
||||
|
||||
|
||||
def _modes_present(tokens: Sequence) -> list[str]:
|
||||
"""Every mode any token declares, base first then the rest alphabetically."""
|
||||
modes = {
|
||||
mode
|
||||
for token in tokens
|
||||
for mode in (getattr(token, "value_by_mode", None) or {})
|
||||
}
|
||||
rest = sorted(modes - {BASE_MODE})
|
||||
return ([BASE_MODE] if BASE_MODE in modes else []) + rest
|
||||
|
||||
|
||||
def render_stylesheet(
|
||||
tokens: Sequence,
|
||||
*,
|
||||
root_selector: str = ":root",
|
||||
title: str = "",
|
||||
design_system_id: int | None = None,
|
||||
) -> str:
|
||||
"""The master sheet for a resolved token set.
|
||||
|
||||
One block per mode. Within a block, only the tokens that declare a value for
|
||||
that mode — so a mode block is an override layer, exactly as the storage
|
||||
model has it.
|
||||
|
||||
Tokens with no value at all are emitted as COMMENTED-OUT declarations in
|
||||
their group, not dropped. The rulebook named them, so their absence is a
|
||||
finding, and a commented line puts that finding where the reader is already
|
||||
looking.
|
||||
"""
|
||||
header = [
|
||||
"/*",
|
||||
f" * {safe_comment(title) or 'Design system'} — generated stylesheet",
|
||||
]
|
||||
if design_system_id is not None:
|
||||
header.append(f" * Source: design system {int(design_system_id)}.")
|
||||
header += [
|
||||
" *",
|
||||
" * Purpose tokens only. This sheet declares what values MEAN; it styles",
|
||||
" * no elements. Buttons, tables and input schemes live as snippets that",
|
||||
" * reference these names, so each value is stated once and reused rather",
|
||||
" * than restated per element.",
|
||||
" *",
|
||||
" * Generated — edit the design system, not this file.",
|
||||
" */",
|
||||
"",
|
||||
]
|
||||
|
||||
lines = list(header)
|
||||
valueless = [
|
||||
t for t in tokens
|
||||
if is_valid_token_name(getattr(t, "name", ""))
|
||||
and not (getattr(t, "value_by_mode", None) or {})
|
||||
]
|
||||
|
||||
for mode in _modes_present(tokens):
|
||||
block: list[str] = []
|
||||
for group, group_tokens in _grouped(tokens):
|
||||
entries: list[str] = []
|
||||
for token in group_tokens:
|
||||
name = (getattr(token, "name", "") or "").strip()
|
||||
if not is_valid_token_name(name):
|
||||
continue
|
||||
raw = (getattr(token, "value_by_mode", None) or {}).get(mode)
|
||||
if raw is None:
|
||||
continue
|
||||
value = safe_value(str(raw))
|
||||
if value is None:
|
||||
entries.append(
|
||||
f" /* {name}: value rejected — not a safe CSS value */"
|
||||
)
|
||||
continue
|
||||
# Purpose first — what the token is FOR is what a reader of the
|
||||
# stylesheet needs. Rationale is the fallback so a token that
|
||||
# only carries the why still says something.
|
||||
purpose = safe_comment(
|
||||
getattr(token, "purpose", "")
|
||||
or getattr(token, "rationale", "")
|
||||
or ""
|
||||
)
|
||||
comment = f" /* {purpose} */" if purpose and mode == BASE_MODE else ""
|
||||
entries.append(f" {name}: {value};{comment}")
|
||||
if entries:
|
||||
if group:
|
||||
block.append(f" /* {safe_comment(group)} */")
|
||||
block.extend(entries)
|
||||
block.append("")
|
||||
|
||||
# Declared-but-valueless tokens belong to the base layer: they have no
|
||||
# mode to sit under, and repeating them per mode would triple the noise.
|
||||
if mode == BASE_MODE and valueless:
|
||||
block.append(" /* Declared by the design system, no value set yet: */")
|
||||
block.extend(f" /* {t.name}: ; */" for t in valueless)
|
||||
block.append("")
|
||||
|
||||
if not block:
|
||||
continue
|
||||
lines.append(f"{selector_for_mode(mode, root_selector)} {{")
|
||||
lines.extend(block[:-1] if block[-1] == "" else block)
|
||||
lines.append("}")
|
||||
lines.append("")
|
||||
|
||||
# A trailing, machine-readable record of what this system says to write
|
||||
# INSTEAD of a given literal.
|
||||
#
|
||||
# The sheet carries its own supersedes declarations so that any consumer has
|
||||
# them — notably a CI check, which has the component sources but no database.
|
||||
# Hardcoding the mapping in a checker would bake one install's palette into
|
||||
# the tool; reading it from the sheet keeps the checker instance-agnostic and
|
||||
# keeps this file the single source.
|
||||
replacements = [
|
||||
(literal, getattr(token, "name", ""))
|
||||
for token in tokens
|
||||
for literal in (getattr(token, "supersedes", None) or ())
|
||||
if is_valid_token_name(getattr(token, "name", ""))
|
||||
]
|
||||
if replacements:
|
||||
lines.append("/* SUPERSEDES — write the token, not the literal.")
|
||||
for literal, name in replacements:
|
||||
lines.append(f" * {safe_comment(str(literal))} -> {name}")
|
||||
lines.append(" */")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def duplicate_values(tokens: Sequence) -> dict[str, list[str]]:
|
||||
"""Values declared by more than one token, mapped to the names declaring them.
|
||||
|
||||
Two names for one value are either a deliberate alias or the same idea
|
||||
recorded twice — the token-level form of the duplicated-definition shape.
|
||||
Reported rather than refused: a design system legitimately aligns colours on
|
||||
purpose (one palette entry defined as equal to another), and a generator
|
||||
that rejected that
|
||||
would be wrong about the operator's intent.
|
||||
|
||||
Compares the BASE value only. Two tokens agreeing in one mode and diverging
|
||||
in another are not duplicates of each other — they are a near-miss, which is
|
||||
a different and less interesting finding.
|
||||
"""
|
||||
by_value: dict[str, list[str]] = {}
|
||||
for token in tokens:
|
||||
name = getattr(token, "name", "")
|
||||
base = (getattr(token, "value_by_mode", None) or {}).get(BASE_MODE)
|
||||
if not name or not base:
|
||||
continue
|
||||
by_value.setdefault(str(base).strip().lower(), []).append(name)
|
||||
return {value: names for value, names in by_value.items() if len(names) > 1}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reading a sheet from the other side: does this code use it correctly?
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# "The snippets use the tags from the sheet" is a verifiable relation, and
|
||||
# nothing checked it before. Three questions, each a different failure:
|
||||
#
|
||||
# var(--x) where no --x exists -> renders as NOTHING. No error, no test
|
||||
# failure, no visual clue beyond the thing
|
||||
# silently not being styled.
|
||||
# a superseded literal in code -> the value the sheet said to stop writing,
|
||||
# and the sheet knows what to write instead.
|
||||
# --x: declared inside a snippet -> a component minting its own token is the
|
||||
# bloat a shared sheet exists to prevent.
|
||||
#
|
||||
# The first is not hypothetical: `--color-accent` was used throughout a new view
|
||||
# in this codebase and does not exist.
|
||||
|
||||
_VAR_REFERENCE = re.compile(r"var\(\s*(--[A-Za-z0-9_-]+)")
|
||||
_LOCAL_DEFINITION = re.compile(r"(?<![\w-])(--[A-Za-z0-9_-]+)\s*:")
|
||||
|
||||
|
||||
def referenced_tokens(code: str) -> set[str]:
|
||||
"""Every custom property the code reads through `var()`."""
|
||||
return set(_VAR_REFERENCE.findall(code or ""))
|
||||
|
||||
|
||||
def defined_tokens(code: str) -> set[str]:
|
||||
"""Every custom property the code declares itself.
|
||||
|
||||
Excludes names it also reads: `--x: var(--x, fallback)` is a redeclaration
|
||||
of something the sheet owns, which the unknown-reference check already
|
||||
covers more precisely.
|
||||
"""
|
||||
return set(_LOCAL_DEFINITION.findall(code or "")) - referenced_tokens(code or "")
|
||||
|
||||
|
||||
def _literal_pattern(literal: str) -> re.Pattern:
|
||||
"""Match a literal value without matching a longer one that contains it.
|
||||
|
||||
`#fff` must not match inside `#ffffff` — they are different colours, and a
|
||||
finding that fired on the wrong one would send someone to change code that
|
||||
was already correct.
|
||||
"""
|
||||
escaped = re.escape(literal)
|
||||
lead = r"(?<![0-9A-Za-z_#-])"
|
||||
trail = r"(?![0-9A-Za-z_-])" if literal.startswith("#") else r"(?![0-9A-Za-z_-])"
|
||||
return re.compile(lead + escaped + trail, re.IGNORECASE)
|
||||
|
||||
|
||||
def check_code_against_tokens(code: str, tokens) -> dict:
|
||||
"""What this code gets wrong about that token set.
|
||||
|
||||
`tokens` is any sequence with `.name`, `.value_by_mode` and `.supersedes` —
|
||||
resolved tokens, or stored ones.
|
||||
|
||||
Reports rather than scores. Every finding here has a legitimate exception:
|
||||
a snippet may target a system it isn't being checked against, and a literal
|
||||
may be deliberate in a context the token doesn't cover. What it removes is
|
||||
the SILENCE — all three currently fail with no signal at all.
|
||||
"""
|
||||
known = {getattr(t, "name", "") for t in tokens}
|
||||
referenced = referenced_tokens(code)
|
||||
|
||||
superseded: list[dict] = []
|
||||
for token in tokens:
|
||||
for literal in getattr(token, "supersedes", None) or ():
|
||||
if _literal_pattern(str(literal)).search(code or ""):
|
||||
superseded.append({
|
||||
"literal": literal,
|
||||
"use_instead": getattr(token, "name", ""),
|
||||
})
|
||||
|
||||
return {
|
||||
"used": sorted(referenced & known),
|
||||
"unknown": sorted(referenced - known),
|
||||
"superseded_literals": superseded,
|
||||
"local_definitions": sorted(defined_tokens(code)),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Derivation — tokens whose value is a formula over other tokens
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# `--fs-accent-soft: color-mix(in srgb, var(--fs-accent) 15%, transparent)` needs
|
||||
# NO special storage: it is a value like any other, and the browser resolves the
|
||||
# `var()` at use time. Change `--fs-accent` and every derived token shifts with
|
||||
# it, in every mode, from one declaration.
|
||||
#
|
||||
# That last part is the real win. A derived token declared once in the base layer
|
||||
# follows its source through dark mode automatically, because `var()` resolves in
|
||||
# whatever context it is used rather than where it is written. Storing a computed
|
||||
# literal instead would need one row per mode AND would silently stop tracking
|
||||
# the source the moment the source changed.
|
||||
#
|
||||
# What derivation DOES need is the check below. A formula pointing at a token
|
||||
# that does not exist is invalid-at-computed-value-time: the browser drops the
|
||||
# declaration and the element falls back to inheritance or nothing. Silent, like
|
||||
# everything else in this family.
|
||||
|
||||
|
||||
def token_dependencies(tokens) -> dict[str, set[str]]:
|
||||
"""Each token name mapped to the token names its own values reference."""
|
||||
deps: dict[str, set[str]] = {}
|
||||
for token in tokens:
|
||||
name = getattr(token, "name", "")
|
||||
if not name:
|
||||
continue
|
||||
refs: set[str] = set()
|
||||
for value in (getattr(token, "value_by_mode", None) or {}).values():
|
||||
refs |= referenced_tokens(str(value))
|
||||
deps[name] = refs - {name}
|
||||
return deps
|
||||
|
||||
|
||||
def _find_cycles(deps: dict[str, set[str]]) -> list[list[str]]:
|
||||
"""Derivation loops, each reported once as the names involved.
|
||||
|
||||
CSS degrades a loop to invalid-at-computed-value-time rather than hanging, so
|
||||
this is about telling the operator, not about protecting the renderer. A
|
||||
token that quietly resolves to nothing is the failure worth naming.
|
||||
"""
|
||||
cycles: list[list[str]] = []
|
||||
seen_cycles: set[frozenset] = set()
|
||||
|
||||
def walk(node: str, path: list[str], visiting: set[str]) -> None:
|
||||
for dep in sorted(deps.get(node, ())):
|
||||
if dep in visiting:
|
||||
loop = path[path.index(dep):]
|
||||
key = frozenset(loop)
|
||||
if loop and key not in seen_cycles:
|
||||
seen_cycles.add(key)
|
||||
cycles.append(loop)
|
||||
continue
|
||||
if dep in deps:
|
||||
walk(dep, path + [dep], visiting | {dep})
|
||||
|
||||
for name in sorted(deps):
|
||||
walk(name, [name], {name})
|
||||
return cycles
|
||||
|
||||
|
||||
def derivation_report(tokens) -> dict:
|
||||
"""Which tokens are formulas, and which of those are broken.
|
||||
|
||||
`derived` name -> the tokens it is computed from
|
||||
`unknown_refs` name -> references that resolve to no token in this system.
|
||||
The browser drops such a declaration entirely; nothing errors.
|
||||
`cycles` derivation loops, which resolve to nothing for the same reason
|
||||
"""
|
||||
deps = token_dependencies(tokens)
|
||||
known = set(deps)
|
||||
|
||||
derived = {name: sorted(refs) for name, refs in deps.items() if refs}
|
||||
unknown = {
|
||||
name: sorted(refs - known)
|
||||
for name, refs in deps.items()
|
||||
if refs - known
|
||||
}
|
||||
return {
|
||||
"derived": derived,
|
||||
"unknown_refs": unknown,
|
||||
"cycles": _find_cycles(deps),
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
"""Design system + token persistence, and the guard on the parent chain.
|
||||
|
||||
Access goes through `services/access.py`, where reaching a system via
|
||||
a project you can see grants READ but never write — see
|
||||
`get_design_system_permission` for why that asymmetry exists.
|
||||
|
||||
The cascade itself lives in `services/design_cascade.py` as pure functions. This
|
||||
module is the part that needs a database: loading the shape of the hierarchy,
|
||||
refusing writes that would break it, and storing tokens.
|
||||
|
||||
Nothing here seeds or implies a default system. An install with no design
|
||||
systems is an ordinary install, and every caller must handle an empty list as
|
||||
the normal case rather than a missing prerequisite.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.design_system import DesignSystem, DesignToken
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.services import access
|
||||
from grimoire.services.design_stylesheet import (
|
||||
check_code_against_tokens,
|
||||
derivation_report,
|
||||
duplicate_values,
|
||||
render_stylesheet,
|
||||
)
|
||||
from grimoire.services.design_starter_roles import (
|
||||
DEFAULT_TOKEN_PREFIX,
|
||||
starter_tokens,
|
||||
)
|
||||
from grimoire.services.design_cascade import (
|
||||
ResolvedToken,
|
||||
ancestry,
|
||||
resolve_tokens,
|
||||
would_cycle,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DesignSystemCycle(ValueError):
|
||||
"""A parent assignment that would close an inheritance loop.
|
||||
|
||||
Raised rather than returned as None because the two outcomes need different
|
||||
answers: None already means "not found, or not yours", and a caller that
|
||||
conflated them would show "no such design system" for what is really "that
|
||||
parent is one of its own descendants". Routes map this to a 400.
|
||||
"""
|
||||
|
||||
|
||||
async def _parent_map(session, owner_user_id: int) -> dict[int, int | None]:
|
||||
"""`{id: parent_id}` for one owner's live systems — the hierarchy's shape.
|
||||
|
||||
Scoped to the OWNER of the systems, not the caller, and the distinction is
|
||||
load-bearing on the read path: a caller reading through a shared project
|
||||
does not own any link in the chain, and a caller-scoped map would hand them
|
||||
an empty forest and a cascade truncated to one system. They would get a page
|
||||
that renders with plausible wrong values and no error anywhere.
|
||||
|
||||
Owner-scoping is also the constraint on parenting: a chain may only be built
|
||||
from systems its owner controls, or someone else's delete or re-parent would
|
||||
silently restyle your app.
|
||||
"""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(DesignSystem.id, DesignSystem.parent_id).where(
|
||||
DesignSystem.owner_user_id == owner_user_id,
|
||||
DesignSystem.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return dict(rows)
|
||||
|
||||
|
||||
# --- design systems ---------------------------------------------------------
|
||||
|
||||
async def create_design_system(
|
||||
user_id: int,
|
||||
title: str,
|
||||
description: str | None = None,
|
||||
guidance: str | None = None,
|
||||
parent_id: int | None = None,
|
||||
starter_role_groups: list[str] | None = None,
|
||||
token_prefix: str = DEFAULT_TOKEN_PREFIX,
|
||||
) -> DesignSystem | None:
|
||||
"""Create a system, with or without a parent.
|
||||
|
||||
Returns None when `parent_id` names a system the caller may not write —
|
||||
which, per the ACL, means one they do not own.
|
||||
|
||||
`starter_role_groups` seeds the system with named, VALUELESS token roles
|
||||
— the moment a role is missing is the moment a literal gets written
|
||||
instead, so the cheapest time to name them is now. Pass a list of group
|
||||
names to choose, `[]` for none, or None for none.
|
||||
|
||||
None and `[]` deliberately mean the same thing here, unlike in
|
||||
`starter_tokens` where None means "all": creation must not seed 40 rows
|
||||
into a system whose caller never asked. Opting in is the caller's job, and
|
||||
the UI's default of everything-checked lives in the UI.
|
||||
"""
|
||||
if parent_id is not None and not await access.can_write_design_system(
|
||||
user_id, parent_id
|
||||
):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
system = DesignSystem(
|
||||
owner_user_id=user_id,
|
||||
title=title.strip(),
|
||||
description=description,
|
||||
guidance=guidance,
|
||||
parent_id=parent_id,
|
||||
)
|
||||
session.add(system)
|
||||
await session.commit()
|
||||
await session.refresh(system)
|
||||
|
||||
if starter_role_groups:
|
||||
for row in starter_tokens(starter_role_groups, prefix=token_prefix):
|
||||
session.add(DesignToken(design_system_id=system.id, **row))
|
||||
await session.commit()
|
||||
return system
|
||||
|
||||
|
||||
async def get_design_system(user_id: int, design_system_id: int) -> DesignSystem | None:
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
if not await access.can_read_design_system(user_id, design_system_id):
|
||||
return None
|
||||
return system
|
||||
|
||||
|
||||
async def list_design_systems(user_id: int) -> list[DesignSystem]:
|
||||
"""The caller's own systems, ordered by title. Empty is normal."""
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(DesignSystem)
|
||||
.where(
|
||||
DesignSystem.owner_user_id == user_id,
|
||||
DesignSystem.deleted_at.is_(None),
|
||||
)
|
||||
.order_by(DesignSystem.title)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def update_design_system(
|
||||
user_id: int, design_system_id: int, **fields: object
|
||||
) -> DesignSystem | None:
|
||||
"""Partial update. Raises DesignSystemCycle if `parent_id` would loop.
|
||||
|
||||
`parent_id` is handled apart from the other fields because None is a
|
||||
meaningful value for it — "make this a root" — where for every other field
|
||||
None means "leave alone". Callers signal it by passing the key at all.
|
||||
"""
|
||||
if not await access.can_write_design_system(user_id, design_system_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
|
||||
if "parent_id" in fields:
|
||||
parent_id = fields.pop("parent_id")
|
||||
if parent_id is not None:
|
||||
parent_id = int(parent_id)
|
||||
if not await access.can_write_design_system(user_id, parent_id):
|
||||
return None
|
||||
if would_cycle(
|
||||
design_system_id,
|
||||
parent_id,
|
||||
await _parent_map(session, system.owner_user_id),
|
||||
):
|
||||
raise DesignSystemCycle(
|
||||
f"Design system {design_system_id} cannot inherit from "
|
||||
f"{parent_id}: that system already inherits from it."
|
||||
)
|
||||
system.parent_id = parent_id
|
||||
|
||||
for key, value in fields.items():
|
||||
if key in ("title", "description", "guidance") and value is not None:
|
||||
setattr(system, key, value)
|
||||
system.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(system)
|
||||
return system
|
||||
|
||||
|
||||
async def delete_design_system(user_id: int, design_system_id: int) -> bool:
|
||||
"""Soft-delete a system. Children survive as roots (the FK is SET NULL)."""
|
||||
if not await access.can_write_design_system(user_id, design_system_id):
|
||||
return False
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return False
|
||||
system.deleted_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
# --- tokens -----------------------------------------------------------------
|
||||
|
||||
async def create_token(
|
||||
user_id: int,
|
||||
design_system_id: int,
|
||||
name: str,
|
||||
value_by_mode: dict | None = None,
|
||||
group_name: str | None = None,
|
||||
purpose: str | None = None,
|
||||
rationale: str | None = None,
|
||||
supersedes: list | None = None,
|
||||
order_index: int = 0,
|
||||
) -> DesignToken | None:
|
||||
if not await access.can_write_design_system(user_id, design_system_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
token = DesignToken(
|
||||
design_system_id=design_system_id,
|
||||
name=name.strip(),
|
||||
# `or {}` and not the argument as given: the column is NOT NULL so
|
||||
# that absence has exactly one spelling. Passing None here would
|
||||
# otherwise store JSON null and reintroduce the second empty state.
|
||||
value_by_mode=value_by_mode or {},
|
||||
group_name=group_name,
|
||||
purpose=purpose,
|
||||
rationale=rationale,
|
||||
# `or []` for the same reason as value_by_mode above: the column is
|
||||
# NOT NULL so absence has one spelling, and None would store JSON
|
||||
# null instead of an empty array.
|
||||
supersedes=supersedes or [],
|
||||
order_index=order_index,
|
||||
)
|
||||
session.add(token)
|
||||
await session.commit()
|
||||
await session.refresh(token)
|
||||
return token
|
||||
|
||||
|
||||
async def list_tokens(user_id: int, design_system_id: int) -> list[DesignToken]:
|
||||
"""One system's OWN tokens — its override set, not its effective set.
|
||||
|
||||
Resolving the chain is `resolve_design_system`'s job; this deliberately answers the narrower
|
||||
question ("what does this system change?") that the model exists to make
|
||||
free.
|
||||
"""
|
||||
if not await access.can_read_design_system(user_id, design_system_id):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(DesignToken)
|
||||
.where(
|
||||
DesignToken.design_system_id == design_system_id,
|
||||
DesignToken.deleted_at.is_(None),
|
||||
)
|
||||
.order_by(DesignToken.order_index.asc(), DesignToken.name.asc())
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def resolve_design_system(
|
||||
user_id: int, design_system_id: int
|
||||
) -> list[ResolvedToken] | None:
|
||||
"""A system's EFFECTIVE token set — everything it inherits, with its own on top.
|
||||
|
||||
None when the caller may not read the system; an empty list when the chain
|
||||
genuinely holds no tokens, which is an ordinary state for a system that has
|
||||
just been created.
|
||||
|
||||
Two queries regardless of how deep the chain runs: one for the hierarchy's
|
||||
shape, one for every token in it. The flattening itself is
|
||||
`design_cascade.resolve_tokens` — pure, so the cascade rule is tested
|
||||
without a database and this function is only the loading.
|
||||
"""
|
||||
if not await access.can_read_design_system(user_id, design_system_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
|
||||
parents = await _parent_map(session, system.owner_user_id)
|
||||
chain = ancestry(design_system_id, parents)
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(DesignToken)
|
||||
.where(
|
||||
DesignToken.design_system_id.in_(chain),
|
||||
DesignToken.deleted_at.is_(None),
|
||||
)
|
||||
.order_by(DesignToken.order_index.asc(), DesignToken.name.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
tokens_by_system: dict[int, list[DesignToken]] = {}
|
||||
for token in rows:
|
||||
tokens_by_system.setdefault(token.design_system_id, []).append(token)
|
||||
return resolve_tokens(design_system_id, parents, tokens_by_system)
|
||||
|
||||
|
||||
async def design_context(user_id: int, design_system_id: int) -> dict | None:
|
||||
"""What a session needs to know about a design system, before it writes UI.
|
||||
|
||||
This is the DELIVERY side of a design system, and it exists because storing
|
||||
one does not make a session aware of it. Rules get pushed into every session
|
||||
by the plugin's SessionStart hook; a design system had no such channel, so
|
||||
the standards were reachable only by an agent that already knew to go
|
||||
looking — which is the same silent failure as a token nobody declares.
|
||||
|
||||
Guidance is chain-merged, ANCESTOR-FIRST, and that is the point rather than
|
||||
a convenience. A child system holds only what it CHANGES, so its own
|
||||
guidance describes a departure from a house style it never restates. Hand an
|
||||
agent the leaf alone and it builds against a fragment, with no signal that
|
||||
the rest exists.
|
||||
|
||||
Tokens are summarised, not listed: the count and the group names are enough
|
||||
to know what the system covers, and the full set is one call away. Sending
|
||||
a hundred token values into every session start would crowd out the context
|
||||
it is meant to inform.
|
||||
|
||||
None when the caller may not read the system.
|
||||
"""
|
||||
tokens = await resolve_design_system(user_id, design_system_id)
|
||||
if tokens is None:
|
||||
return None
|
||||
|
||||
async with async_session() as session:
|
||||
system = await session.get(DesignSystem, design_system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
parents = await _parent_map(session, system.owner_user_id)
|
||||
chain = ancestry(design_system_id, parents)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(DesignSystem).where(DesignSystem.id.in_(chain))
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
by_id = {s.id: s for s in rows}
|
||||
return {
|
||||
"id": system.id,
|
||||
"title": system.title,
|
||||
"description": system.description or "",
|
||||
# Outermost ancestor first, so the reader meets the house style before
|
||||
# the app's departures from it.
|
||||
"inherits_from": [
|
||||
by_id[sid].title for sid in reversed(chain[1:]) if sid in by_id
|
||||
],
|
||||
"guidance": [
|
||||
{
|
||||
"design_system_id": sid,
|
||||
"title": by_id[sid].title,
|
||||
"guidance": (by_id[sid].guidance or "").strip(),
|
||||
}
|
||||
for sid in reversed(chain)
|
||||
if sid in by_id and (by_id[sid].guidance or "").strip()
|
||||
],
|
||||
"token_count": len(tokens),
|
||||
"token_groups": sorted({t.group_name for t in tokens if t.group_name}),
|
||||
}
|
||||
|
||||
|
||||
async def update_token(
|
||||
user_id: int, token_id: int, **fields: object
|
||||
) -> DesignToken | None:
|
||||
allowed = {
|
||||
"name", "value_by_mode", "group_name", "purpose", "rationale",
|
||||
"supersedes", "order_index",
|
||||
}
|
||||
async with async_session() as session:
|
||||
token = await session.get(DesignToken, token_id)
|
||||
if token is None or token.deleted_at is not None:
|
||||
return None
|
||||
if not await access.can_write_design_system(user_id, token.design_system_id):
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if key in allowed and value is not None:
|
||||
setattr(token, key, value)
|
||||
token.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(token)
|
||||
return token
|
||||
|
||||
|
||||
async def delete_token(user_id: int, token_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
token = await session.get(DesignToken, token_id)
|
||||
if token is None or token.deleted_at is not None:
|
||||
return False
|
||||
if not await access.can_write_design_system(user_id, token.design_system_id):
|
||||
return False
|
||||
token.deleted_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
# --- the project pointer ----------------------------------------------------
|
||||
|
||||
async def set_project_design_system(
|
||||
user_id: int, project_id: int, design_system_id: int | None
|
||||
) -> bool:
|
||||
"""Point a project at a design system, or at nothing (None clears it).
|
||||
|
||||
Needs write on the project and READ on the system: pointing at a system is
|
||||
consuming it, not changing it, so a system shared with you through another
|
||||
project is a legitimate choice here.
|
||||
"""
|
||||
if not await access.can_write_project(user_id, project_id):
|
||||
return False
|
||||
if design_system_id is not None and not await access.can_read_design_system(
|
||||
user_id, design_system_id
|
||||
):
|
||||
return False
|
||||
async with async_session() as session:
|
||||
project = await session.get(Project, project_id)
|
||||
if project is None or project.deleted_at is not None:
|
||||
return False
|
||||
project.design_system_id = design_system_id
|
||||
project.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
# --- the master sheet -------------------------------------------------------
|
||||
|
||||
async def stylesheet_for_system(
|
||||
user_id: int, design_system_id: int, root_selector: str = ":root"
|
||||
) -> dict | None:
|
||||
"""The master CSS sheet a design system generates, plus its reuse report.
|
||||
|
||||
Purpose tokens only — the sheet styles no elements. See
|
||||
`services/design_stylesheet.py` for why that split is the design rather than
|
||||
a shortcut.
|
||||
|
||||
Returns None if the caller may not read the system. The `duplicates` half is
|
||||
advisory: two tokens sharing a value are either a deliberate alias or one
|
||||
idea recorded twice, and only the operator knows which.
|
||||
"""
|
||||
resolved = await resolve_design_system(user_id, design_system_id)
|
||||
if resolved is None:
|
||||
return None
|
||||
system = await get_design_system(user_id, design_system_id)
|
||||
|
||||
return {
|
||||
"design_system_id": design_system_id,
|
||||
"css": render_stylesheet(
|
||||
resolved,
|
||||
root_selector=root_selector,
|
||||
title=system.title if system else "",
|
||||
design_system_id=design_system_id,
|
||||
),
|
||||
"token_count": len(resolved),
|
||||
"valueless": [t.name for t in resolved if not t.value_by_mode],
|
||||
"duplicates": duplicate_values(resolved),
|
||||
# Formulas: which tokens are computed from others, and which of those
|
||||
# point at nothing. A broken formula is dropped by the browser without
|
||||
# any error, so the sheet cannot show it for itself.
|
||||
"derivation": derivation_report(resolved),
|
||||
}
|
||||
|
||||
|
||||
async def check_snippets_against_system(
|
||||
user_id: int, design_system_id: int, project_id: int = 0
|
||||
) -> dict | None:
|
||||
"""Which recorded snippets disagree with this design system's sheet.
|
||||
|
||||
The relation the operator named — "the snippets use the tags from the sheet"
|
||||
— turned into a check. For each snippet: `var(--x)` references with no such
|
||||
token, literals the sheet says to stop writing, and custom properties the
|
||||
snippet mints for itself instead of using shared ones.
|
||||
|
||||
Snippets with nothing to report are omitted entirely. A list of everything
|
||||
that is fine is a list nobody reads twice.
|
||||
"""
|
||||
resolved = await resolve_design_system(user_id, design_system_id)
|
||||
if resolved is None:
|
||||
return None
|
||||
|
||||
from grimoire.services import snippets as snippets_svc
|
||||
|
||||
# `list_snippets` returns (rows, total) and caps limit at 100; `project_id`
|
||||
# must be None — not 0 — to reach across every project, since 0 would filter
|
||||
# to a project with that id.
|
||||
rows, _total = await snippets_svc.list_snippets(
|
||||
user_id=user_id, project_id=project_id or None, limit=100,
|
||||
)
|
||||
|
||||
findings: list[dict] = []
|
||||
for row in rows:
|
||||
snippet_id = row.get("id")
|
||||
if snippet_id is None:
|
||||
continue
|
||||
# The list rows carry a preview, not the code. The check has to read the
|
||||
# whole body or it would report on a truncation.
|
||||
note = await snippets_svc.get_snippet(user_id=user_id, snippet_id=int(snippet_id))
|
||||
if note is None:
|
||||
continue
|
||||
report = check_code_against_tokens(note.body or "", resolved)
|
||||
if not (
|
||||
report["unknown"] or report["superseded_literals"] or report["local_definitions"]
|
||||
):
|
||||
continue
|
||||
findings.append({
|
||||
"snippet_id": int(snippet_id),
|
||||
"title": note.title or "",
|
||||
**report,
|
||||
})
|
||||
|
||||
return {
|
||||
"design_system_id": design_system_id,
|
||||
"checked": len(rows),
|
||||
"findings": findings,
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Lightweight diagnostic instrumentation for crash investigation.
|
||||
|
||||
The Grimoire app + its Postgres have been crashing recurrently with no
|
||||
clear cause in the logs. This module adds three things designed to make
|
||||
the crash class identifiable from logs alone:
|
||||
|
||||
1. **Heartbeat** — once per minute, log a snapshot of process resources
|
||||
(RSS memory, asyncio task count, DB pool checked-in/out). A sudden
|
||||
silence in heartbeats lets you bound the crash time to within a
|
||||
minute, and the last snapshot before silence usually rules in or out
|
||||
memory growth / pool exhaustion.
|
||||
|
||||
2. **Signal handler** — catches SIGTERM and SIGINT, logs them with the
|
||||
sender's intent ("docker stop", "swarm restart", "manual ctrl-C")
|
||||
then lets the normal shutdown proceed. Distinguishes orderly
|
||||
shutdown from kill-9 / OOM-kill (which can't be caught and will
|
||||
show as a silent log gap followed by container exit code 137).
|
||||
|
||||
3. **Asyncio exception hook** — every Task that raises an uncaught
|
||||
exception logs a full traceback. Without this, `asyncio.create_task`
|
||||
exceptions are swallowed silently — the chat crash that locked us
|
||||
into 409 forever was exactly this pattern.
|
||||
|
||||
All three are read-only / log-only — no behavior changes. Safe to
|
||||
leave running in production indefinitely; the cost is one log line
|
||||
per minute and ~0.1ms of work per heartbeat.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Heartbeat cadence. 60s is the sweet spot: short enough that a crash
|
||||
# window is bounded to a useful interval, long enough that the log
|
||||
# noise is negligible. If we ever need more resolution during active
|
||||
# debugging, drop it temporarily to 15s.
|
||||
_HEARTBEAT_INTERVAL_SECS = 60
|
||||
|
||||
# Persistent diagnostic state — written to /data so it survives
|
||||
# container restart, OOM-kill, and Docker log rotation. The point of
|
||||
# the persistence: the operator may not notice a crash for hours, and
|
||||
# by then the Docker logs are gone. /data/diagnostics gives them a
|
||||
# durable place to look post-mortem.
|
||||
_DIAG_DIR = Path("/data/diagnostics")
|
||||
_CURRENT_STATE_PATH = _DIAG_DIR / "current.json"
|
||||
_LAST_SHUTDOWN_PATH = _DIAG_DIR / "last_shutdown.json"
|
||||
_LAST_EXCEPTION_PATH = _DIAG_DIR / "last_exception.json"
|
||||
_PREVIOUS_RUN_PATH = _DIAG_DIR / "previous_run.json"
|
||||
_DIAG_LOG_PATH = _DIAG_DIR / "diag.log"
|
||||
|
||||
# Dedicated file logger for the heartbeat/exception/signal stream.
|
||||
# Separate from the stdout logger so it can't be lost by Docker log
|
||||
# rotation. Rotates at 10 MB, keeps 5 backups → 50 MB max footprint.
|
||||
_file_logger: logging.Logger | None = None
|
||||
|
||||
_heartbeat_task: asyncio.Task | None = None
|
||||
_shutdown_logged = False # don't double-log shutdown if multiple signals arrive
|
||||
_started_at: float | None = None
|
||||
|
||||
|
||||
def _process_rss_mb() -> float | None:
|
||||
"""Resident-set memory in MB. Read from /proc/self/status — no deps."""
|
||||
try:
|
||||
with open("/proc/self/status") as fh:
|
||||
for line in fh:
|
||||
if line.startswith("VmRSS:"):
|
||||
# Format: 'VmRSS: 123456 kB'
|
||||
kb = int(line.split()[1])
|
||||
return round(kb / 1024, 1)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _db_pool_stats() -> dict[str, Any]:
|
||||
"""Pool checked-in / checked-out / overflow. Direct from SQLAlchemy."""
|
||||
try:
|
||||
from grimoire.models import engine
|
||||
pool = engine.pool
|
||||
# Async engines wrap a sync pool; .checkedin() / .checkedout() exist
|
||||
# on the underlying QueuePool. Attribute access is documented but
|
||||
# version-fragile, so wrap in try.
|
||||
return {
|
||||
"size": getattr(pool, "size", lambda: None)(),
|
||||
"checked_in": getattr(pool, "checkedin", lambda: None)(),
|
||||
"checked_out": getattr(pool, "checkedout", lambda: None)(),
|
||||
"overflow": getattr(pool, "overflow", lambda: None)(),
|
||||
}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _asyncio_task_count() -> int | None:
|
||||
try:
|
||||
return len([t for t in asyncio.all_tasks() if not t.done()])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _uptime_secs() -> float | None:
|
||||
if _started_at is None:
|
||||
return None
|
||||
return round(time.monotonic() - _started_at, 1)
|
||||
|
||||
|
||||
def _snapshot_dict(reason: str = "heartbeat") -> dict[str, Any]:
|
||||
"""Bundle the current resource snapshot into a JSON-safe dict.
|
||||
|
||||
This is the canonical 'what was the app doing right now' record —
|
||||
written to current.json every heartbeat, to last_shutdown.json on
|
||||
signal, to last_exception.json on uncaught task exception. The
|
||||
`reason` field tells you which event captured this snapshot.
|
||||
"""
|
||||
return {
|
||||
"reason": reason,
|
||||
"wall_time_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"uptime_secs": _uptime_secs(),
|
||||
"rss_mb": _process_rss_mb(),
|
||||
"asyncio_tasks": _asyncio_task_count(),
|
||||
"db_pool": _db_pool_stats(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
|
||||
|
||||
def _write_state_atomic(path: Path, data: dict[str, Any]) -> None:
|
||||
"""Write JSON to `path` via a tmp+rename so a crash mid-write can't
|
||||
leave a half-written / unparseable file. The tmp file is on the same
|
||||
filesystem as the target so the rename is atomic.
|
||||
|
||||
Silent on failure — diagnostic writes must never raise into the
|
||||
caller. If /data isn't writable, the in-memory + stdout logging
|
||||
still happens.
|
||||
"""
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2, default=str))
|
||||
tmp.replace(path)
|
||||
except Exception:
|
||||
# Log to stdout — if this fires, /data is bad and the operator
|
||||
# needs to know. Don't propagate; diagnostics must not crash
|
||||
# the heartbeat that depends on them.
|
||||
logger.exception("Failed to write diagnostic state to %s", path)
|
||||
|
||||
|
||||
def _setup_file_logger() -> logging.Logger:
|
||||
"""Create or return the dedicated rotating file logger for diagnostics.
|
||||
|
||||
Separate from the app's stdout logger so an aggressive log rotator
|
||||
or Docker log cleanup can't take it out. Writes to /data/diagnostics/diag.log
|
||||
with size-based rotation.
|
||||
"""
|
||||
global _file_logger
|
||||
if _file_logger is not None:
|
||||
return _file_logger
|
||||
try:
|
||||
_DIAG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
flog = logging.getLogger("grimoire.diagnostics.persistent")
|
||||
flog.setLevel(logging.INFO)
|
||||
flog.propagate = False # don't double-log into the root stdout stream
|
||||
# Don't re-add handlers across module reloads / re-runs.
|
||||
if not flog.handlers:
|
||||
handler = RotatingFileHandler(
|
||||
_DIAG_LOG_PATH,
|
||||
maxBytes=10 * 1024 * 1024,
|
||||
backupCount=5,
|
||||
)
|
||||
handler.setFormatter(
|
||||
logging.Formatter(
|
||||
"%(asctime)s %(levelname)s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
)
|
||||
flog.addHandler(handler)
|
||||
_file_logger = flog
|
||||
except Exception:
|
||||
logger.exception("Failed to set up persistent diagnostic logger")
|
||||
# Fall back to the regular stdout logger so callers don't need
|
||||
# to null-check the return value.
|
||||
_file_logger = logger
|
||||
return _file_logger
|
||||
|
||||
|
||||
def _persistent_log(level: int, msg: str, *args: Any) -> None:
|
||||
"""Emit one line to BOTH the stdout logger AND the persistent file."""
|
||||
logger.log(level, msg, *args)
|
||||
try:
|
||||
_setup_file_logger().log(level, msg, *args)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _post_mortem_previous_run() -> None:
|
||||
"""On startup, check the persistent state and tell the operator if
|
||||
the previous run died abruptly.
|
||||
|
||||
Logic: if current.json exists and is newer than last_shutdown.json,
|
||||
then either there was never a clean shutdown (first ever run, or
|
||||
SIGKILL/OOM/abrupt termination of the previous run). If current.json
|
||||
exists AND last_shutdown.json doesn't, OR current.json's timestamp
|
||||
is newer than last_shutdown.json's → the previous run didn't get
|
||||
to write its shutdown record. Log the last-known state prominently
|
||||
so the operator notices on next visit.
|
||||
"""
|
||||
try:
|
||||
if not _CURRENT_STATE_PATH.exists():
|
||||
return # first ever run
|
||||
|
||||
current_mtime = _CURRENT_STATE_PATH.stat().st_mtime
|
||||
shutdown_mtime = (
|
||||
_LAST_SHUTDOWN_PATH.stat().st_mtime
|
||||
if _LAST_SHUTDOWN_PATH.exists() else 0
|
||||
)
|
||||
if shutdown_mtime >= current_mtime:
|
||||
return # previous run shut down cleanly after its last heartbeat
|
||||
|
||||
# The previous run had a heartbeat without a subsequent clean
|
||||
# shutdown — that's a crash, OOM, or otherwise-killed run.
|
||||
try:
|
||||
last_snapshot = json.loads(_CURRENT_STATE_PATH.read_text())
|
||||
except Exception:
|
||||
last_snapshot = {"_parse_error": "current.json unreadable"}
|
||||
|
||||
seconds_since = round(time.time() - current_mtime, 1)
|
||||
_persistent_log(
|
||||
logging.WARNING,
|
||||
"diag post-mortem: PREVIOUS RUN DIED ABRUPTLY. Last heartbeat "
|
||||
"was %ss before this startup. Last-known state: %s",
|
||||
seconds_since, json.dumps(last_snapshot),
|
||||
)
|
||||
|
||||
# Preserve the offending snapshot for retrospection. This file
|
||||
# accumulates one rename per abrupt termination so the user can
|
||||
# diff sequences over time if it's a recurring pattern.
|
||||
try:
|
||||
_PREVIOUS_RUN_PATH.write_text(json.dumps({
|
||||
"noticed_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"seconds_since_last_heartbeat": seconds_since,
|
||||
"had_shutdown_record": _LAST_SHUTDOWN_PATH.exists(),
|
||||
"had_exception_record": _LAST_EXCEPTION_PATH.exists(),
|
||||
"last_known_state": last_snapshot,
|
||||
}, indent=2, default=str))
|
||||
except Exception:
|
||||
logger.exception("Failed to preserve previous_run.json")
|
||||
except Exception:
|
||||
logger.exception("Post-mortem check itself failed")
|
||||
|
||||
|
||||
async def _heartbeat_loop() -> None:
|
||||
"""Forever-running task that emits one snapshot per interval.
|
||||
|
||||
Each heartbeat does three things:
|
||||
1. Logs the snapshot to stdout (Docker logs — short-lived).
|
||||
2. Appends the snapshot to /data/diagnostics/diag.log (rotating;
|
||||
survives container restart).
|
||||
3. Atomically rewrites /data/diagnostics/current.json with the
|
||||
latest snapshot — so post-crash you can `cat` one file and see
|
||||
the last known good state instead of scanning the rolling log.
|
||||
|
||||
Exceptions inside the loop are caught and logged so the loop itself
|
||||
can't die silently — the whole point is that this thing keeps
|
||||
talking even when other things crash around it.
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
snap = _snapshot_dict(reason="heartbeat")
|
||||
_persistent_log(
|
||||
logging.INFO,
|
||||
"diag heartbeat: uptime=%ss rss=%sMB asyncio_tasks=%s "
|
||||
"db_pool=%s",
|
||||
snap["uptime_secs"], snap["rss_mb"], snap["asyncio_tasks"],
|
||||
snap["db_pool"],
|
||||
)
|
||||
_write_state_atomic(_CURRENT_STATE_PATH, snap)
|
||||
except Exception:
|
||||
logger.exception("Heartbeat snapshot crashed (continuing)")
|
||||
try:
|
||||
await asyncio.sleep(_HEARTBEAT_INTERVAL_SECS)
|
||||
except asyncio.CancelledError:
|
||||
_persistent_log(
|
||||
logging.INFO, "diag heartbeat: shutting down (CancelledError)"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def _asyncio_exception_handler(loop: asyncio.AbstractEventLoop, context: dict) -> None:
|
||||
"""Log unhandled task exceptions instead of letting them disappear.
|
||||
|
||||
Default asyncio behaviour: if a fire-and-forget task raises and
|
||||
nothing awaits the result, the exception is logged at ERROR but the
|
||||
surrounding context (which task, what coroutine) can be sparse.
|
||||
This handler enriches the log line so we can tell WHICH task crashed.
|
||||
"""
|
||||
msg = context.get("message", "")
|
||||
exc = context.get("exception")
|
||||
task = context.get("task") or context.get("future")
|
||||
task_name = getattr(task, "get_name", lambda: "?")() if task else "?"
|
||||
coro = getattr(task, "get_coro", lambda: None)() if task else None
|
||||
coro_name = getattr(coro, "__qualname__", str(coro)) if coro else "?"
|
||||
|
||||
if exc is not None:
|
||||
logger.error(
|
||||
"asyncio unhandled exception in task %r (coro=%s): %s",
|
||||
task_name, coro_name, msg,
|
||||
exc_info=(type(exc), exc, exc.__traceback__),
|
||||
)
|
||||
# Also emit to the persistent file logger so the traceback
|
||||
# survives even if Docker logs are flushed.
|
||||
try:
|
||||
_setup_file_logger().error(
|
||||
"asyncio unhandled exception in task %r (coro=%s): %s",
|
||||
task_name, coro_name, msg,
|
||||
exc_info=(type(exc), exc, exc.__traceback__),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# Persist the snapshot at exception time so post-mortem can see
|
||||
# what the app's state was when it crashed.
|
||||
snap = _snapshot_dict(reason="asyncio_exception")
|
||||
snap["task_name"] = task_name
|
||||
snap["coro_name"] = coro_name
|
||||
snap["exception_type"] = type(exc).__name__
|
||||
snap["exception_message"] = str(exc)
|
||||
_write_state_atomic(_LAST_EXCEPTION_PATH, snap)
|
||||
else:
|
||||
logger.error(
|
||||
"asyncio unhandled event in task %r (coro=%s): %s — context=%r",
|
||||
task_name, coro_name, msg, context,
|
||||
)
|
||||
|
||||
|
||||
def _signal_handler(loop: asyncio.AbstractEventLoop, signame: str) -> None:
|
||||
"""Catch SIGTERM/SIGINT and log them.
|
||||
|
||||
Doesn't try to interfere with shutdown — Hypercorn handles its own
|
||||
graceful exit. We just want a log line so we can tell "orderly
|
||||
shutdown via signal X" apart from "silent gap then container exit"
|
||||
(which means kill-9 or OOM, neither of which is catchable).
|
||||
"""
|
||||
global _shutdown_logged
|
||||
if _shutdown_logged:
|
||||
return
|
||||
_shutdown_logged = True
|
||||
snap = _snapshot_dict(reason="signal_shutdown")
|
||||
snap["signal"] = signame
|
||||
_persistent_log(
|
||||
logging.WARNING,
|
||||
"diag shutdown: received %s, expecting graceful exit. snapshot=%s",
|
||||
signame, json.dumps(snap),
|
||||
)
|
||||
# Write before-exit snapshot so the next startup's post-mortem can
|
||||
# see this run shut down cleanly via signal (rather than crashed).
|
||||
_write_state_atomic(_LAST_SHUTDOWN_PATH, snap)
|
||||
|
||||
|
||||
def start_diagnostics(loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Install all three diagnostic surfaces. Idempotent."""
|
||||
global _heartbeat_task, _started_at
|
||||
|
||||
if _started_at is None:
|
||||
_started_at = time.monotonic()
|
||||
|
||||
# 0. Set up the persistent file logger FIRST, then run the
|
||||
# post-mortem check. The post-mortem reads /data state from the
|
||||
# previous run and surfaces "previous run died abruptly" if
|
||||
# current.json is newer than last_shutdown.json. Operator catches
|
||||
# this on next visit even if they missed the crash window.
|
||||
_setup_file_logger()
|
||||
_post_mortem_previous_run()
|
||||
|
||||
# 1. Asyncio exception hook — install once.
|
||||
if loop.get_exception_handler() is None:
|
||||
loop.set_exception_handler(_asyncio_exception_handler)
|
||||
|
||||
# 2. Signal handlers. add_signal_handler is Unix-only; skip on
|
||||
# Windows so dev on a non-Linux machine doesn't blow up.
|
||||
if os.name == "posix":
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(
|
||||
sig,
|
||||
_signal_handler,
|
||||
loop,
|
||||
sig.name,
|
||||
)
|
||||
except (RuntimeError, NotImplementedError, ValueError):
|
||||
# add_signal_handler can fail when not running in the main
|
||||
# thread or when the loop is already managing the signal.
|
||||
# Either case is fine — heartbeat + exception hook still work.
|
||||
pass
|
||||
|
||||
# 3. Heartbeat loop. Start if not already running (idempotent on
|
||||
# restart-style reloads).
|
||||
if _heartbeat_task is None or _heartbeat_task.done():
|
||||
_heartbeat_task = asyncio.create_task(
|
||||
_heartbeat_loop(), name="diag-heartbeat",
|
||||
)
|
||||
logger.info(
|
||||
"diag started: heartbeat every %ds, signal-aware shutdown logging on, "
|
||||
"asyncio exception hook installed",
|
||||
_HEARTBEAT_INTERVAL_SECS,
|
||||
)
|
||||
|
||||
|
||||
def stop_diagnostics() -> None:
|
||||
"""Cancel the heartbeat. Called from after_serving.
|
||||
|
||||
Lets the loop emit one last 'shutting down' log line so the silence
|
||||
that follows is intentional (not a crash). The exception hook stays
|
||||
installed but won't fire after the loop closes.
|
||||
"""
|
||||
global _heartbeat_task
|
||||
if _heartbeat_task is not None and not _heartbeat_task.done():
|
||||
_heartbeat_task.cancel()
|
||||
_heartbeat_task = None
|
||||
snap = _snapshot_dict(reason="after_serving_shutdown")
|
||||
_persistent_log(
|
||||
logging.INFO,
|
||||
"diag stopped: heartbeat cancelled. Final snapshot=%s",
|
||||
json.dumps(snap),
|
||||
)
|
||||
# If we got here without the signal handler firing (e.g., Hypercorn
|
||||
# decided to shut down on its own), still record a clean exit so
|
||||
# the next startup's post-mortem sees a fresh last_shutdown.json
|
||||
# and doesn't flag this as an abrupt death.
|
||||
_write_state_atomic(_LAST_SHUTDOWN_PATH, snap)
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Email service for sending notifications via SMTP."""
|
||||
|
||||
import logging
|
||||
from email.message import EmailMessage
|
||||
|
||||
import aiosmtplib
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.config import Config
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.setting import Setting
|
||||
from grimoire.models.user import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Inline SVG logo for email (white palette, renders on indigo header)
|
||||
_EMAIL_LOGO_SVG = (
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32"'
|
||||
' style="display:inline-block;vertical-align:middle;margin-right:8px;">'
|
||||
'<path d="M4 7 C4 7 8 5 16 6 C24 5 28 7 28 7 L28 26 C28 26 24 24 16 25 C8 24 4 26 4 26 Z"'
|
||||
' fill="#ffffff" stroke="#c4b5fd" stroke-width="0.5"/>'
|
||||
'<line x1="16" y1="6" x2="16" y2="25" stroke="#c4b5fd" stroke-width="0.8"/>'
|
||||
'<line x1="7" y1="11" x2="14" y2="10.5" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="7" y1="14" x2="14" y2="13.5" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="7" y1="17" x2="14" y2="16.5" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="7" y1="20" x2="12" y2="19.5" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="18" y1="10.5" x2="25" y2="11" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="18" y1="13.5" x2="25" y2="14" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<line x1="18" y1="16.5" x2="25" y2="17" stroke="#ddd6fe" stroke-width="0.6" stroke-linecap="round"/>'
|
||||
'<g transform="translate(24, 5)">'
|
||||
'<path d="M0 -4 L1 -1 L4 0 L1 1 L0 4 L-1 1 L-4 0 L-1 -1 Z" fill="#f6ad55"/>'
|
||||
'<path d="M0 -2.5 L0.6 -0.6 L2.5 0 L0.6 0.6 L0 2.5 L-0.6 0.6 L-2.5 0 L-0.6 -0.6 Z" fill="#fbd38d"/>'
|
||||
'</g>'
|
||||
'<circle cx="20" cy="3" r="0.7" fill="#f6ad55" opacity="0.7"/>'
|
||||
'<circle cx="27" cy="8" r="0.5" fill="#fbd38d" opacity="0.6"/>'
|
||||
'</svg>'
|
||||
)
|
||||
|
||||
|
||||
def _email_html(title: str, body: str) -> str:
|
||||
"""Wrap email body content in the standard Fabled Grimoire template."""
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="color-scheme" content="light">
|
||||
</head>
|
||||
<body style="margin:0;padding:0;background:#f5f3ff;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif;">
|
||||
<div style="padding:36px 16px 48px;">
|
||||
<div style="max-width:520px;margin:0 auto;">
|
||||
|
||||
<!-- Card -->
|
||||
<div style="background:#ffffff;border:1px solid #ddd6fe;border-radius:14px;overflow:hidden;box-shadow:0 4px 24px rgba(109,40,217,0.08);">
|
||||
|
||||
<!-- Header -->
|
||||
<div style="background:linear-gradient(135deg,#7c3aed 0%,#6d28d9 100%);padding:24px 28px;text-align:center;">
|
||||
<div style="margin-bottom:8px;">
|
||||
{_EMAIL_LOGO_SVG}<span style="display:inline-block;vertical-align:middle;color:#ffffff;font-size:18px;font-weight:700;letter-spacing:0.01em;">Fabled Grimoire</span>
|
||||
</div>
|
||||
<p style="margin:0;color:#ede9fe;font-size:13px;letter-spacing:0.03em;">{title}</p>
|
||||
</div>
|
||||
|
||||
<!-- Body -->
|
||||
<div style="padding:32px 28px;color:#1e1b4b;">
|
||||
{body}
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div style="border-top:1px solid #ede9fe;padding:16px 28px;text-align:center;background:#faf5ff;">
|
||||
<p style="margin:0;color:#a78bfa;font-size:12px;">Sent by your Fabled Grimoire instance.</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
SMTP_SETTING_KEYS = [
|
||||
"smtp_host",
|
||||
"smtp_port",
|
||||
"smtp_username",
|
||||
"smtp_password",
|
||||
"smtp_from_address",
|
||||
"smtp_from_name",
|
||||
"smtp_use_tls",
|
||||
]
|
||||
|
||||
|
||||
async def get_smtp_config() -> dict[str, str]:
|
||||
"""Get SMTP config from admin's settings, falling back to env vars."""
|
||||
config: dict[str, str] = {
|
||||
"smtp_host": Config.SMTP_HOST,
|
||||
"smtp_port": str(Config.SMTP_PORT),
|
||||
"smtp_username": Config.SMTP_USERNAME,
|
||||
"smtp_password": Config.SMTP_PASSWORD,
|
||||
"smtp_from_address": Config.SMTP_FROM_ADDRESS,
|
||||
"smtp_from_name": Config.SMTP_FROM_NAME,
|
||||
"smtp_use_tls": "true" if Config.SMTP_USE_TLS else "false",
|
||||
}
|
||||
|
||||
# Override with DB settings from admin user
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting)
|
||||
.join(User, Setting.user_id == User.id)
|
||||
.where(User.role == "admin", Setting.key.in_(SMTP_SETTING_KEYS))
|
||||
)
|
||||
for setting in result.scalars().all():
|
||||
config[setting.key] = setting.value
|
||||
|
||||
return config
|
||||
|
||||
|
||||
async def is_smtp_configured() -> bool:
|
||||
"""Check if SMTP is configured (has a host set)."""
|
||||
config = await get_smtp_config()
|
||||
return bool(config.get("smtp_host"))
|
||||
|
||||
|
||||
async def get_base_url() -> str:
|
||||
"""Get the application base URL from admin settings, falling back to Config.BASE_URL."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting)
|
||||
.join(User, Setting.user_id == User.id)
|
||||
.where(User.role == "admin", Setting.key == "base_url")
|
||||
)
|
||||
setting = result.scalars().first()
|
||||
if setting and setting.value:
|
||||
return setting.value.rstrip("/")
|
||||
return Config.BASE_URL.rstrip("/")
|
||||
|
||||
|
||||
async def send_email(to: str, subject: str, html_body: str) -> None:
|
||||
"""Send an email via SMTP."""
|
||||
config = await get_smtp_config()
|
||||
host = config.get("smtp_host")
|
||||
if not host:
|
||||
logger.debug("SMTP not configured, skipping email to %s", to)
|
||||
return
|
||||
|
||||
port = int(config.get("smtp_port", "587"))
|
||||
username = config.get("smtp_username", "")
|
||||
password = config.get("smtp_password", "")
|
||||
from_address = config.get("smtp_from_address", "")
|
||||
from_name = config.get("smtp_from_name", "Fabled Grimoire")
|
||||
use_tls = config.get("smtp_use_tls", "true") == "true"
|
||||
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = f"{from_name} <{from_address}>" if from_name else from_address
|
||||
msg["To"] = to
|
||||
msg.set_content(subject) # plain text fallback
|
||||
msg.add_alternative(html_body, subtype="html")
|
||||
|
||||
try:
|
||||
await aiosmtplib.send(
|
||||
msg,
|
||||
hostname=host,
|
||||
port=port,
|
||||
username=username or None,
|
||||
password=password or None,
|
||||
start_tls=use_tls and port != 465,
|
||||
use_tls=port == 465,
|
||||
)
|
||||
logger.info("Email sent to %s: %s", to, subject)
|
||||
except Exception:
|
||||
logger.exception("Failed to send email to %s: %s", to, subject)
|
||||
raise
|
||||
|
||||
|
||||
async def send_test_email(to: str) -> None:
|
||||
"""Send a branded test email."""
|
||||
body = """
|
||||
<p style="margin:0 0 12px;color:#1e1b4b;font-size:15px;font-weight:600;">SMTP is configured correctly</p>
|
||||
<p style="margin:0;color:#6b7280;font-size:14px;">Your Fabled Grimoire instance can send email notifications.</p>
|
||||
"""
|
||||
await send_email(to, "Fabled Grimoire - Test Email", _email_html("Test Email", body))
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,496 @@
|
||||
"""Forge adapter — optional server-side READ access to the operator's git forge.
|
||||
|
||||
The recorded location of a snippet is the source of truth for its code and
|
||||
the stored body is a cache; this module is the seam that lets the SERVER read
|
||||
that source of truth, so the cache can be refreshed at pull time, drift can be
|
||||
flagged from push webhooks, and coverage can be measured.
|
||||
|
||||
Design constraints, in force everywhere below:
|
||||
|
||||
- OPTIONAL per user. Connections are
|
||||
per-user keyring rows resolved by repo host on the PROJECT OWNER's
|
||||
keyring; `get_forges()` returns an empty selector when the owner has
|
||||
nothing configured, and every consumer must treat that as "behave as if
|
||||
there were no forge". A user who never configures a forge is not degraded — that is
|
||||
the baseline.
|
||||
- READ-ONLY by construction. The adapter exposes reads; there is no write
|
||||
method to misuse. The token an operator mints for it only ever needs read
|
||||
scope, and the docs say so.
|
||||
- The contract stays as small as its consumers: read_file /
|
||||
latest_commit / archive / default_branch / resolve_repo / check. Two
|
||||
implementations (Gitea, GitHub) keep it honest; resist widening
|
||||
it speculatively.
|
||||
- Repo identity is the repo-binding key — `normalize_repo_key`'s
|
||||
host/owner/repo — so the join between a snippet's recorded repo and the
|
||||
forge needs no new identity scheme. The host segment selects whether THIS
|
||||
forge can serve the repo; the remainder is the API path.
|
||||
- Errors carry no token, ever, and failures are exceptions the caller
|
||||
handles — a consumer decides whether to fall back (pull-time fetch) or
|
||||
surface (settings test button); this module never silently swallows.
|
||||
|
||||
This is also the codebase's first outbound-HTTP client with a real timeout
|
||||
convention (oauth.py predates it): short total timeout, no retries — every
|
||||
consumer has a fallback, so a slow forge must cost bounded time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.config import Config
|
||||
from grimoire.services.repo_bindings import normalize_repo_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Kinds a connection can use. Matches _FORGE_CLASSES below.
|
||||
FORGE_KINDS = ("gitea", "github")
|
||||
|
||||
# Total budget per forge call. Consumers either have a cache to fall back to
|
||||
# or a user watching a button (the test probe) — neither tolerates a
|
||||
# hung socket, and there is no retry: the fallback IS the retry policy.
|
||||
_TIMEOUT = httpx.Timeout(5.0)
|
||||
|
||||
# Archive downloads move a whole-repo tarball and only ever run off the
|
||||
# request path (coverage recompute), so they get a bigger budget than
|
||||
# the per-file reads — but still a bound, because a hung background task
|
||||
# holds a connection slot as surely as a foreground one.
|
||||
_ARCHIVE_TIMEOUT = httpx.Timeout(60.0)
|
||||
|
||||
|
||||
class ForgeError(RuntimeError):
|
||||
"""A forge call failed (network, auth, unexpected payload). Token-free."""
|
||||
|
||||
|
||||
class ForgeNotFound(ForgeError):
|
||||
"""The repo, path, or ref does not exist on the forge — the one failure
|
||||
consumers treat differently, because for a recorded snippet location it is
|
||||
itself a finding (the recorded path is gone)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ForgeFile:
|
||||
"""One file read from the forge at a specific point in history."""
|
||||
|
||||
content: str
|
||||
# The commit the content was served at — what provenance stores.
|
||||
commit_sha: str
|
||||
path: str
|
||||
|
||||
|
||||
def host_of(url: str) -> str:
|
||||
"""The lowercase hostname of a URL — the keyring's lookup key."""
|
||||
return (urlsplit(url).hostname or "").lower()
|
||||
|
||||
|
||||
class ForgeAdapter:
|
||||
"""The shared plumbing of the forge contract; adapters supply the API
|
||||
base, auth headers, and any endpoint that differs.
|
||||
|
||||
`transport` exists for tests: httpx.MockTransport makes the contract
|
||||
testable without a live server or a new dependency. Production callers
|
||||
never pass it.
|
||||
"""
|
||||
|
||||
kind = ""
|
||||
# One "newest commit for this path" page — the endpoint is shared but the
|
||||
# page-size parameter is not, so each adapter names its own.
|
||||
_commit_page_params: dict = {}
|
||||
|
||||
def __init__(self, base_url: str, token: str, *, transport=None) -> None:
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
self._token = token or ""
|
||||
self._transport = transport
|
||||
|
||||
@property
|
||||
def host(self) -> str:
|
||||
return host_of(self.base_url)
|
||||
|
||||
def resolve_repo(self, repo_or_url: str) -> str | None:
|
||||
"""The forge-API repo path for a recorded repo — or None if this forge
|
||||
does not serve it.
|
||||
|
||||
Accepts anything `normalize_repo_key` accepts (a raw remote URL or an
|
||||
already-normalized key). None is a NORMAL answer, not an error: a
|
||||
snippet recorded against github.com on an instance whose forge is a
|
||||
self-hosted Gitea is simply out of this forge's reach.
|
||||
"""
|
||||
key = normalize_repo_key(repo_or_url or "")
|
||||
if not key or "/" not in key:
|
||||
return None
|
||||
host, _, rest = key.partition("/")
|
||||
if host != self.host or "/" not in rest:
|
||||
return None
|
||||
return rest
|
||||
|
||||
def _api_base(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def _headers(self) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
kwargs: dict = {
|
||||
"base_url": self._api_base(),
|
||||
"headers": self._headers(),
|
||||
"timeout": _TIMEOUT,
|
||||
# GitHub serves tarballs via a 302 to codeload. httpx drops the
|
||||
# Authorization header on the cross-host hop, and GitHub's
|
||||
# redirect target carries its own short-lived token in the URL —
|
||||
# so following is both necessary there and harmless on Gitea.
|
||||
"follow_redirects": True,
|
||||
}
|
||||
if self._transport is not None:
|
||||
kwargs["transport"] = self._transport
|
||||
return httpx.AsyncClient(**kwargs)
|
||||
|
||||
async def _get(self, client: httpx.AsyncClient, url: str, **kw) -> httpx.Response:
|
||||
try:
|
||||
resp = await client.get(url, **kw)
|
||||
except httpx.HTTPError as exc:
|
||||
# str(exc) on transport errors names hosts and timeouts, never
|
||||
# headers — safe, and the detail is what makes the test button useful.
|
||||
raise ForgeError(f"forge unreachable: {exc}") from exc
|
||||
if resp.status_code == 404:
|
||||
raise ForgeNotFound(f"not found on forge: {url}")
|
||||
if resp.status_code in (401, 403):
|
||||
raise ForgeError("forge rejected the token (check its read scope)")
|
||||
if resp.status_code >= 400:
|
||||
raise ForgeError(f"forge returned HTTP {resp.status_code} for {url}")
|
||||
return resp
|
||||
|
||||
def _decode_contents(self, payload, path: str) -> str:
|
||||
"""Both forges speak the same contents-API dialect: a base64 file
|
||||
object, a list for a directory."""
|
||||
if isinstance(payload, list):
|
||||
raise ForgeNotFound(f"{path} is a directory on the forge, not a file")
|
||||
if payload.get("type") != "file":
|
||||
raise ForgeNotFound(
|
||||
f"{path} is a {payload.get('type', 'non-file')} on the forge"
|
||||
)
|
||||
if payload.get("encoding") != "base64" or payload.get("content") is None:
|
||||
raise ForgeError(f"forge returned no readable content for {path}")
|
||||
try:
|
||||
return base64.b64decode(payload["content"]).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError) as exc:
|
||||
raise ForgeError(f"forge content for {path} is not utf-8 text") from exc
|
||||
|
||||
async def _newest_commit(
|
||||
self, client: httpx.AsyncClient, repo: str, path: str, ref: str
|
||||
) -> str:
|
||||
params: dict = {**self._commit_page_params, "path": path}
|
||||
if ref:
|
||||
params["sha"] = ref
|
||||
resp = await self._get(client, f"/repos/{repo}/commits", params=params)
|
||||
payload = resp.json()
|
||||
# Tolerant parse on purpose: the caller uses this as an optimization
|
||||
# and falls back to read_file, so a surprising payload must read as
|
||||
# "don't know", never break a pull.
|
||||
if isinstance(payload, list) and payload and isinstance(payload[0], dict):
|
||||
return str(payload[0].get("sha") or "")
|
||||
return ""
|
||||
|
||||
async def latest_commit(self, repo: str, path: str, ref: str = "") -> str:
|
||||
"""The newest commit touching ``path`` — "" when it can't be told.
|
||||
|
||||
The cached-SHA short-circuit: when a snippet's provenance
|
||||
already names a commit, this one small call can prove the file
|
||||
hasn't moved since — no content transfer, which is what keeps
|
||||
pull-time freshness inside GitHub's rate limits.
|
||||
"""
|
||||
async with self._client() as client:
|
||||
return await self._newest_commit(client, repo, path, ref)
|
||||
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
async def archive(self, repo: str, ref: str) -> bytes:
|
||||
"""The repo's content at ``ref`` as a gzipped tarball, in one request.
|
||||
|
||||
Coverage measurement needs every source file's text; per-file
|
||||
reads would mean one API call per file, so the archive endpoint is the
|
||||
only shape that scales past toy repos. Callers must never run this in
|
||||
a request path — it moves the whole repo.
|
||||
"""
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client, self._archive_url(repo, ref), timeout=_ARCHIVE_TIMEOUT
|
||||
)
|
||||
return resp.content
|
||||
|
||||
async def default_branch(self, repo: str) -> str:
|
||||
async with self._client() as client:
|
||||
resp = await self._get(client, f"/repos/{repo}")
|
||||
branch = (resp.json() or {}).get("default_branch") or ""
|
||||
if not branch:
|
||||
raise ForgeError(f"forge reported no default branch for {repo}")
|
||||
return branch
|
||||
|
||||
|
||||
class GiteaForge(ForgeAdapter):
|
||||
"""The Gitea implementation of the forge contract, over its REST API."""
|
||||
|
||||
kind = "gitea"
|
||||
# stat/verification/files add per-commit work Gitea skips when told to.
|
||||
_commit_page_params = {"limit": 1, "stat": "false"}
|
||||
|
||||
def _api_base(self) -> str:
|
||||
return f"{self.base_url}/api/v1"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {"Authorization": f"token {self._token}"}
|
||||
|
||||
async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile:
|
||||
"""Read one file's current content, with the commit it was served at.
|
||||
|
||||
`repo` is the API path from resolve_repo ("owner/repo"); `ref` is a
|
||||
branch, tag, or commit — empty means the default branch.
|
||||
"""
|
||||
params = {"ref": ref} if ref else None
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client,
|
||||
f"/repos/{repo}/contents/{quote(path, safe='/')}",
|
||||
params=params,
|
||||
)
|
||||
payload = resp.json()
|
||||
content = self._decode_contents(payload, path)
|
||||
return ForgeFile(
|
||||
content=content,
|
||||
# last_commit_sha is the commit that last touched the file — the
|
||||
# honest provenance stamp. The blob sha is a content address, not
|
||||
# a point in history, so it is deliberately not surfaced.
|
||||
commit_sha=payload.get("last_commit_sha") or "",
|
||||
path=payload.get("path") or path,
|
||||
)
|
||||
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
return f"/repos/{repo}/archive/{quote(ref, safe='')}.tar.gz"
|
||||
|
||||
async def check(self) -> dict:
|
||||
"""Health probe for the settings test button: reach the forge AND
|
||||
prove the token is accepted. Returns {"ok", "version", "username"}."""
|
||||
async with self._client() as client:
|
||||
version = (await self._get(client, "/version")).json() or {}
|
||||
user = (await self._get(client, "/user")).json() or {}
|
||||
return {
|
||||
"ok": True,
|
||||
"version": version.get("version") or "",
|
||||
"username": user.get("login") or user.get("username") or "",
|
||||
}
|
||||
|
||||
|
||||
class GitHubForge(ForgeAdapter):
|
||||
"""The GitHub implementation — the second one, which is the point: it
|
||||
proves the seam is a contract rather than a Gitea-shaped hole. Works
|
||||
against github.com and GitHub Enterprise; the token is a fine-grained PAT
|
||||
with Contents: Read-only (or a classic token with `repo` read)."""
|
||||
|
||||
kind = "github"
|
||||
_commit_page_params = {"per_page": 1}
|
||||
|
||||
_API_VERSION = "2022-11-28"
|
||||
|
||||
def _api_base(self) -> str:
|
||||
# github.com's API lives on its own host; GitHub Enterprise serves
|
||||
# the same API under the instance at /api/v3.
|
||||
if self.host == "github.com":
|
||||
return "https://api.github.com"
|
||||
return f"{self.base_url}/api/v3"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {
|
||||
"Authorization": f"Bearer {self._token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": self._API_VERSION,
|
||||
}
|
||||
|
||||
async def read_file(self, repo: str, path: str, ref: str = "") -> ForgeFile:
|
||||
"""Same contents-API dialect as Gitea, minus one field: GitHub's
|
||||
payload carries only the blob sha — a content address, not a point in
|
||||
history — so the provenance stamp costs one extra commits call. ""
|
||||
when even that can't be told; consumers already treat an empty stamp
|
||||
as "don't restamp"."""
|
||||
params = {"ref": ref} if ref else None
|
||||
async with self._client() as client:
|
||||
resp = await self._get(
|
||||
client,
|
||||
f"/repos/{repo}/contents/{quote(path, safe='/')}",
|
||||
params=params,
|
||||
)
|
||||
payload = resp.json()
|
||||
content = self._decode_contents(payload, path)
|
||||
try:
|
||||
commit_sha = await self._newest_commit(client, repo, path, ref)
|
||||
except ForgeError:
|
||||
commit_sha = ""
|
||||
return ForgeFile(
|
||||
content=content,
|
||||
commit_sha=commit_sha,
|
||||
path=payload.get("path") or path,
|
||||
)
|
||||
|
||||
def _archive_url(self, repo: str, ref: str) -> str:
|
||||
return f"/repos/{repo}/tarball/{quote(ref, safe='')}"
|
||||
|
||||
async def check(self) -> dict:
|
||||
"""GitHub has no /version endpoint; proving the token against /user
|
||||
is the whole probe, and the pinned API version stands in as the
|
||||
version string."""
|
||||
async with self._client() as client:
|
||||
user = (await self._get(client, "/user")).json() or {}
|
||||
return {
|
||||
"ok": True,
|
||||
"version": f"GitHub API {self._API_VERSION}",
|
||||
"username": user.get("login") or "",
|
||||
}
|
||||
|
||||
|
||||
_FORGE_CLASSES: dict[str, type[ForgeAdapter]] = {
|
||||
"gitea": GiteaForge,
|
||||
"github": GitHubForge,
|
||||
}
|
||||
|
||||
|
||||
def _token_of(row) -> str:
|
||||
"""A keyring row's token, opened; empty when this instance can't open it,
|
||||
which build_adapter reads as "this connection cannot serve reads"."""
|
||||
from grimoire.services.forge_connections import connection_token
|
||||
|
||||
token = connection_token(row)
|
||||
if token is None:
|
||||
logger.warning(
|
||||
"forge connection %s (%s) holds a token this instance can't open — "
|
||||
"SECRET_KEY changed since it was saved; its owner must enter it again",
|
||||
row.id, row.host,
|
||||
)
|
||||
return ""
|
||||
return token
|
||||
|
||||
|
||||
def build_adapter(
|
||||
kind: str, base_url: str, token: str, *, transport=None
|
||||
) -> ForgeAdapter | None:
|
||||
"""One validated adapter from raw connection values, or None.
|
||||
|
||||
None means "this connection cannot serve reads", decided per keyring
|
||||
row. Misconfigurations
|
||||
are logged, never raised: a bad row must not break the reads the good
|
||||
rows can still serve.
|
||||
"""
|
||||
kind = (kind or "").strip().lower()
|
||||
base_url = (base_url or "").rstrip("/")
|
||||
cls = _FORGE_CLASSES.get(kind)
|
||||
if cls is None:
|
||||
if kind:
|
||||
# A kind we don't implement is a misconfiguration, not "off" —
|
||||
# say so once per lookup rather than silently reading as absent.
|
||||
logger.warning("unknown forge kind %r configured — connection disabled", kind)
|
||||
return None
|
||||
if not base_url or not token:
|
||||
return None
|
||||
if not base_url.startswith(("http://", "https://")):
|
||||
logger.warning("forge base URL %r has no http(s) scheme — connection disabled", base_url)
|
||||
return None
|
||||
return cls(base_url, token, transport=transport)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ForgeSelector:
|
||||
"""The forge reads available to one project owner.
|
||||
|
||||
Consumers ask it to serve a REPO, not to hand over "the forge": resolve()
|
||||
walks the owner's adapters and returns the (adapter, api_repo) pair for
|
||||
the first one whose host serves the repo — or None, which every consumer
|
||||
treats exactly as "no forge configured". An empty selector is the
|
||||
baseline every install starts from, not a degraded mode.
|
||||
"""
|
||||
|
||||
adapters: tuple[ForgeAdapter, ...] = ()
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.adapters)
|
||||
|
||||
def resolve(self, repo_or_url: str) -> tuple[ForgeAdapter, str] | None:
|
||||
for adapter in self.adapters:
|
||||
repo = adapter.resolve_repo(repo_or_url)
|
||||
if repo is not None:
|
||||
return adapter, repo
|
||||
return None
|
||||
|
||||
|
||||
async def get_forges(
|
||||
owner_id: int, project_id: int | None = None, *, transport=None
|
||||
) -> ForgeSelector:
|
||||
"""The forge selector for reads on behalf of ``owner_id``'s records.
|
||||
|
||||
The keyring model: every server-side forge read for a record runs
|
||||
on the PROJECT OWNER's connections, resolved by repo host — a forge token
|
||||
is a user's credential, and one user's reads must never ride another
|
||||
user's token. Pass the record's ``project_id`` so the per-project pin
|
||||
applies: a pinned project uses ONLY its pinned connection (explicit and
|
||||
auditable); a pin that no longer belongs to the owner (ownership moved) is
|
||||
ignored with a warning rather than honored across users.
|
||||
|
||||
The env config (FORGE_KIND/FORGE_BASE_URL/FORGE_TOKEN) survives as an
|
||||
implicit keyring entry for ADMIN owners only — it is the operator's
|
||||
token, so it must not serve other users' reads — and a stored row for the
|
||||
same host beats it, because the UI writes rows.
|
||||
"""
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.forge_connection import ForgeConnection
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.user import User
|
||||
|
||||
async with async_session() as session:
|
||||
pinned_id = None
|
||||
if project_id:
|
||||
pinned_id = (
|
||||
await session.execute(
|
||||
select(Project.forge_connection_id).where(Project.id == project_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
rows = list(
|
||||
(
|
||||
await session.execute(
|
||||
select(ForgeConnection)
|
||||
.where(ForgeConnection.user_id == owner_id)
|
||||
.order_by(ForgeConnection.id)
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
role = ""
|
||||
if Config.FORGE_KIND and Config.FORGE_BASE_URL and Config.FORGE_TOKEN:
|
||||
role = (
|
||||
await session.execute(select(User.role).where(User.id == owner_id))
|
||||
).scalar_one_or_none() or ""
|
||||
|
||||
if pinned_id:
|
||||
pin = next((r for r in rows if r.id == pinned_id), None)
|
||||
if pin is not None:
|
||||
adapter = build_adapter(pin.kind, pin.base_url, _token_of(pin), transport=transport)
|
||||
return ForgeSelector((adapter,) if adapter is not None else ())
|
||||
logger.warning(
|
||||
"project %s pins forge connection %s the owner (%s) does not hold — pin ignored",
|
||||
project_id, pinned_id, owner_id,
|
||||
)
|
||||
|
||||
adapters: list[ForgeAdapter] = []
|
||||
for row in rows:
|
||||
adapter = build_adapter(row.kind, row.base_url, _token_of(row), transport=transport)
|
||||
if adapter is not None:
|
||||
adapters.append(adapter)
|
||||
if role == "admin":
|
||||
env = build_adapter(
|
||||
Config.FORGE_KIND, Config.FORGE_BASE_URL, Config.FORGE_TOKEN,
|
||||
transport=transport,
|
||||
)
|
||||
if env is not None and all(a.host != env.host for a in adapters):
|
||||
adapters.append(env)
|
||||
return ForgeSelector(tuple(adapters))
|
||||
@@ -0,0 +1,205 @@
|
||||
"""User-level forge connection CRUD — the keyring rows get_forges reads.
|
||||
|
||||
A connection is a user's read-only credential for one forge host; one row per
|
||||
(user, host) keeps host-keyed resolution deterministic with no default-pointer
|
||||
machinery. Everything here is own-rows-only: a connection is a credential, and
|
||||
no caller — admin included — reads or edits another user's. The token never
|
||||
leaves the server (model.to_dict omits it; routes mask "set/unset"), and it is
|
||||
stored sealed (services/secrets.py): a copy of the database alone cannot read
|
||||
it. Every reader opens it through `connection_token`.
|
||||
|
||||
Validation matches what build_adapter will accept, checked here so a bad
|
||||
value is a 400 at the form instead of a silently dead keyring row.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.forge_connection import ForgeConnection
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.services.forge import FORGE_KINDS, host_of
|
||||
from grimoire.services.secrets import seal, unseal
|
||||
|
||||
|
||||
def connection_token(row: ForgeConnection) -> str | None:
|
||||
"""The connection's token in the clear, or None when this instance can't
|
||||
open it — sealed under a different SECRET_KEY (a restore onto a new
|
||||
instance, a rotated key). The owner enters it again; nothing else can."""
|
||||
return unseal(row.token_sealed)
|
||||
|
||||
|
||||
def validate_connection(kind: str, base_url: str) -> str | None:
|
||||
"""The error a connection's non-secret values would earn, or None."""
|
||||
if kind not in FORGE_KINDS:
|
||||
return f"Unknown forge kind {kind!r} (one of: {', '.join(FORGE_KINDS)})"
|
||||
if not base_url.startswith(("http://", "https://")):
|
||||
return "Forge base URL must use http or https"
|
||||
if not host_of(base_url):
|
||||
return "Forge base URL carries no hostname"
|
||||
return None
|
||||
|
||||
|
||||
async def list_connections(user_id: int) -> list[ForgeConnection]:
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(ForgeConnection)
|
||||
.where(ForgeConnection.user_id == user_id)
|
||||
.order_by(ForgeConnection.host)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def get_connection(user_id: int, connection_id: int) -> ForgeConnection | None:
|
||||
async with async_session() as session:
|
||||
return (
|
||||
await session.execute(
|
||||
select(ForgeConnection).where(
|
||||
ForgeConnection.id == connection_id,
|
||||
ForgeConnection.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
async def create_connection(
|
||||
user_id: int, *, kind: str, base_url: str, token: str
|
||||
) -> ForgeConnection:
|
||||
"""Create a keyring row. Raises ValueError on bad values or a host the
|
||||
user already holds — one row per (user, host) IS the resolution model,
|
||||
so a second token for the same host is an update, not a create."""
|
||||
kind = (kind or "").strip().lower()
|
||||
base_url = (base_url or "").strip().rstrip("/")
|
||||
token = token or ""
|
||||
error = validate_connection(kind, base_url)
|
||||
if error is None and not token:
|
||||
error = "A token is required (read scope is enough)"
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
host = host_of(base_url)
|
||||
async with async_session() as session:
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(ForgeConnection).where(
|
||||
ForgeConnection.user_id == user_id,
|
||||
ForgeConnection.host == host,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise ValueError(
|
||||
f"You already have a connection for {host} — edit that one; "
|
||||
"resolution is by host, so a second row could never be reached"
|
||||
)
|
||||
row = ForgeConnection(
|
||||
user_id=user_id, kind=kind, base_url=base_url, host=host,
|
||||
token_sealed=seal(token),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
async def update_connection(
|
||||
user_id: int,
|
||||
connection_id: int,
|
||||
*,
|
||||
kind: str = "",
|
||||
base_url: str = "",
|
||||
token: str = "",
|
||||
) -> ForgeConnection | None:
|
||||
"""Update own row; empty string = leave unchanged (the settings-form
|
||||
sentinel convention). None when the row isn't the caller's."""
|
||||
async with async_session() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(ForgeConnection).where(
|
||||
ForgeConnection.id == connection_id,
|
||||
ForgeConnection.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
new_kind = (kind or "").strip().lower() or row.kind
|
||||
new_base = (base_url or "").strip().rstrip("/") or row.base_url
|
||||
error = validate_connection(new_kind, new_base)
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
new_host = host_of(new_base)
|
||||
if new_host != row.host:
|
||||
clash = (
|
||||
await session.execute(
|
||||
select(ForgeConnection.id).where(
|
||||
ForgeConnection.user_id == user_id,
|
||||
ForgeConnection.host == new_host,
|
||||
ForgeConnection.id != row.id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if clash is not None:
|
||||
raise ValueError(
|
||||
f"You already have a connection for {new_host} — edit that one"
|
||||
)
|
||||
row.kind = new_kind
|
||||
row.base_url = new_base
|
||||
row.host = new_host
|
||||
if token:
|
||||
row.token_sealed = seal(token)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
async def delete_connection(user_id: int, connection_id: int) -> bool:
|
||||
"""Delete own row. Project pins pointing at it go NULL (FK SET NULL) —
|
||||
those projects fall back to keyring resolution, which is the documented
|
||||
unpinned behavior, not a surprise."""
|
||||
async with async_session() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(ForgeConnection).where(
|
||||
ForgeConnection.id == connection_id,
|
||||
ForgeConnection.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def set_project_pin(
|
||||
owner_id: int, project_id: int, connection_id: int | None
|
||||
) -> bool:
|
||||
"""Point a project at one of its OWNER's connections, or clear the pin.
|
||||
|
||||
The caller settles WHO may ask (routes check owner-or-admin); this
|
||||
settles WHOSE connection is eligible: only the project owner's — pinning
|
||||
a collaborator's token to someone else's project is the confused-deputy
|
||||
channel this feature exists to close. False = project or connection not
|
||||
eligible.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
project = (
|
||||
await session.execute(select(Project).where(Project.id == project_id))
|
||||
).scalar_one_or_none()
|
||||
if project is None or (project.user_id or 0) != owner_id:
|
||||
return False
|
||||
if connection_id:
|
||||
held = (
|
||||
await session.execute(
|
||||
select(ForgeConnection.id).where(
|
||||
ForgeConnection.id == connection_id,
|
||||
ForgeConnection.user_id == owner_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if held is None:
|
||||
return False
|
||||
project.forge_connection_id = connection_id or None
|
||||
await session.commit()
|
||||
return True
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Group management service."""
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.group import Group, GroupMembership
|
||||
from grimoire.models.user import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def create_group(
|
||||
user_id: int, name: str, description: str | None = None
|
||||
) -> Group:
|
||||
async with async_session() as session:
|
||||
group = Group(name=name, description=description, created_by=user_id)
|
||||
session.add(group)
|
||||
await session.flush()
|
||||
session.add(GroupMembership(group_id=group.id, user_id=user_id, role="owner"))
|
||||
await session.commit()
|
||||
await session.refresh(group)
|
||||
return group
|
||||
|
||||
|
||||
async def list_groups(user_id: int) -> list[dict]:
|
||||
"""All users see all groups with member count and their own membership status."""
|
||||
async with async_session() as session:
|
||||
groups = (await session.execute(select(Group).order_by(Group.name))).scalars().all()
|
||||
user_group_ids = set(
|
||||
(await session.execute(
|
||||
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
||||
)).scalars().all()
|
||||
)
|
||||
result = []
|
||||
for g in groups:
|
||||
count = len(
|
||||
(await session.execute(
|
||||
select(GroupMembership).where(GroupMembership.group_id == g.id)
|
||||
)).scalars().all()
|
||||
)
|
||||
d = g.to_dict()
|
||||
d["member_count"] = count
|
||||
d["is_member"] = g.id in user_group_ids
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
|
||||
async def get_group(group_id: int) -> Group | None:
|
||||
async with async_session() as session:
|
||||
return await session.get(Group, group_id)
|
||||
|
||||
|
||||
async def _is_group_owner(session, acting_user_id: int, group_id: int) -> bool:
|
||||
m = (await session.execute(
|
||||
select(GroupMembership).where(
|
||||
GroupMembership.group_id == group_id,
|
||||
GroupMembership.user_id == acting_user_id,
|
||||
GroupMembership.role == "owner",
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
return m is not None
|
||||
|
||||
|
||||
async def update_group(
|
||||
acting_user_id: int, group_id: int, is_site_admin: bool, **fields
|
||||
) -> Group | None:
|
||||
async with async_session() as session:
|
||||
group = await session.get(Group, group_id)
|
||||
if not group:
|
||||
return None
|
||||
if not is_site_admin and not await _is_group_owner(session, acting_user_id, group_id):
|
||||
return None
|
||||
for k, v in fields.items():
|
||||
if hasattr(group, k):
|
||||
setattr(group, k, v)
|
||||
await session.commit()
|
||||
await session.refresh(group)
|
||||
return group
|
||||
|
||||
|
||||
async def delete_group(acting_user_id: int, group_id: int, is_site_admin: bool) -> bool:
|
||||
async with async_session() as session:
|
||||
group = await session.get(Group, group_id)
|
||||
if not group:
|
||||
return False
|
||||
if not is_site_admin and group.created_by != acting_user_id:
|
||||
return False
|
||||
await session.delete(group)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_members(group_id: int) -> list[dict]:
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(GroupMembership, User)
|
||||
.join(User, User.id == GroupMembership.user_id)
|
||||
.where(GroupMembership.group_id == group_id)
|
||||
)).all()
|
||||
return [
|
||||
{**m.to_dict(), "username": u.username, "email": u.email}
|
||||
for m, u in rows
|
||||
]
|
||||
|
||||
|
||||
async def add_member(
|
||||
acting_user_id: int,
|
||||
group_id: int,
|
||||
target_user_id: int,
|
||||
role: str,
|
||||
is_site_admin: bool,
|
||||
) -> GroupMembership | None:
|
||||
async with async_session() as session:
|
||||
if not is_site_admin and not await _is_group_owner(session, acting_user_id, group_id):
|
||||
return None
|
||||
membership = GroupMembership(group_id=group_id, user_id=target_user_id, role=role)
|
||||
session.add(membership)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.refresh(membership)
|
||||
return membership
|
||||
|
||||
|
||||
async def update_member_role(
|
||||
acting_user_id: int,
|
||||
group_id: int,
|
||||
target_user_id: int,
|
||||
role: str,
|
||||
is_site_admin: bool,
|
||||
) -> GroupMembership | None:
|
||||
async with async_session() as session:
|
||||
if not is_site_admin and not await _is_group_owner(session, acting_user_id, group_id):
|
||||
return None
|
||||
m = (await session.execute(
|
||||
select(GroupMembership).where(
|
||||
GroupMembership.group_id == group_id,
|
||||
GroupMembership.user_id == target_user_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if not m:
|
||||
return None
|
||||
m.role = role
|
||||
await session.commit()
|
||||
await session.refresh(m)
|
||||
return m
|
||||
|
||||
|
||||
async def remove_member(
|
||||
acting_user_id: int,
|
||||
group_id: int,
|
||||
target_user_id: int,
|
||||
is_site_admin: bool,
|
||||
) -> bool:
|
||||
"""Group owner, site admin, or self-removal are all permitted."""
|
||||
async with async_session() as session:
|
||||
is_self = acting_user_id == target_user_id
|
||||
if not is_site_admin and not is_self:
|
||||
if not await _is_group_owner(session, acting_user_id, group_id):
|
||||
return False
|
||||
m = (await session.execute(
|
||||
select(GroupMembership).where(
|
||||
GroupMembership.group_id == group_id,
|
||||
GroupMembership.user_id == target_user_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if not m:
|
||||
return False
|
||||
await session.delete(m)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def get_user_groups(user_id: int) -> list[Group]:
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(Group)
|
||||
.join(GroupMembership, GroupMembership.group_id == Group.id)
|
||||
.where(GroupMembership.user_id == user_id)
|
||||
)).scalars().all()
|
||||
return list(rows)
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Project inception — what a project was decided to inherit.
|
||||
|
||||
A project's inheritance is a decision, not a default. The record lives on
|
||||
``projects.inception``::
|
||||
|
||||
{
|
||||
"decided_at": "<iso>", "decided_by": <user id> | null,
|
||||
"via": "mcp" | "ui" | "legacy",
|
||||
"choices": {
|
||||
"design_system_id": <id> | null,
|
||||
"seed_systems": bool,
|
||||
"platforms": [<slug>, ...] | null
|
||||
}
|
||||
}
|
||||
|
||||
NULL = undecided → enter_project asks. ``legacy`` is the migration's stamp on
|
||||
projects that predate inception records (inherit-all / no design system /
|
||||
no seed), so the ask fires only for projects created since.
|
||||
|
||||
Rules are not a choice: a rule in a rulebook is global and applies to every
|
||||
project, and a project's own rules are written on it directly. Migration 0101
|
||||
strips the retired ``exclude_always_on_rulebooks`` and ``subscribe_rulebooks``
|
||||
keys from stored records.
|
||||
|
||||
The shape and its validator are pure; ``decide`` composes the existing
|
||||
services — set_project_design_system and the standard Systems seed — checks
|
||||
every target BEFORE touching anything,
|
||||
applies the effects (each idempotent), and writes the record LAST, so a
|
||||
half-applied decision is re-runnable rather than recorded as done.
|
||||
``current_defaults`` is what the enter_project ask shows: what binds today
|
||||
if nobody decides.
|
||||
|
||||
``platforms`` is which platforms the project IS — the answer
|
||||
that decides which family ideas reach it. A list of catalog SLUGS, not ids:
|
||||
the record is JSON, and a slug survives a backup restore onto an install whose
|
||||
ids differ, where an id inside JSON would come back naming another platform.
|
||||
NULL means the question was not answered here, and the project's memberships
|
||||
are left exactly as they are; a list is the full answer — every platform in it
|
||||
is declared, and any platform detection had added that is NOT in it is
|
||||
recorded as a "no", so the next refresh cannot put it back.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.project import Project
|
||||
|
||||
INCEPTION_VIAS = ("mcp", "ui", "legacy")
|
||||
CHOICE_KEYS = ("design_system_id", "seed_systems", "platforms")
|
||||
|
||||
|
||||
def validate_inception(choices) -> str | None:
|
||||
"""The structural error an inception ``choices`` object would earn, or
|
||||
None. Pure and checked BEFORE any effect is applied: a decision either
|
||||
applies whole or errors whole.
|
||||
|
||||
Accepts three keys, each optional: ``design_system_id`` an int or None,
|
||||
``seed_systems`` a bool, ``platforms`` a list of slugs or None. Unknown
|
||||
keys are an error — a typo, or a choice
|
||||
the product no longer offers, must not become a silently ignored one."""
|
||||
if not isinstance(choices, dict):
|
||||
return "choices must be an object"
|
||||
unknown = sorted(set(choices) - set(CHOICE_KEYS))
|
||||
if unknown:
|
||||
return f"unknown inception choice(s): {', '.join(unknown)} (one of: {', '.join(CHOICE_KEYS)})"
|
||||
ds = choices.get("design_system_id")
|
||||
if ds is not None and (isinstance(ds, bool) or not isinstance(ds, int) or ds <= 0):
|
||||
return "design_system_id must be a positive id or null"
|
||||
seed = choices.get("seed_systems", False)
|
||||
if not isinstance(seed, bool):
|
||||
return "seed_systems must be true or false"
|
||||
platforms = choices.get("platforms")
|
||||
if platforms is not None and (
|
||||
not isinstance(platforms, list)
|
||||
or not all(isinstance(p, str) and p.strip() for p in platforms)
|
||||
):
|
||||
return "platforms must be a list of platform slugs (list_platforms), or null"
|
||||
return None
|
||||
|
||||
|
||||
def normalize_choices(choices: dict | None) -> dict:
|
||||
"""Every key, always present, in canonical form — what gets stored
|
||||
and what the UI/agent reads back. Call after validate_inception."""
|
||||
choices = choices or {}
|
||||
platforms = choices.get("platforms")
|
||||
return {
|
||||
"design_system_id": choices.get("design_system_id"),
|
||||
"seed_systems": bool(choices.get("seed_systems", False)),
|
||||
"platforms": (
|
||||
sorted({p.strip() for p in platforms}) if platforms is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def is_decided(project) -> bool:
|
||||
"""A project is decided once its inception record exists (any via)."""
|
||||
return bool(getattr(project, "inception", None))
|
||||
|
||||
|
||||
async def current_defaults(user_id: int, project_id: int) -> dict:
|
||||
"""What the project inherits if nobody decides — the ask's payload.
|
||||
|
||||
{design_system_id, design_systems: [{id,title}], systems: <count>,
|
||||
platforms: [{slug,name}], project_platforms: [{slug,name,state}]}.
|
||||
Instance-agnostic: an install with no design systems shows an empty list,
|
||||
and the ask says so rather than inventing a default. `project_platforms`
|
||||
is what detection has already found (and anything already answered), so
|
||||
the form can start from it.
|
||||
"""
|
||||
from grimoire.services import design_systems as design_systems_svc
|
||||
from grimoire.services import platforms as platforms_svc
|
||||
from grimoire.services import projects as projects_svc
|
||||
from grimoire.services import systems as systems_svc
|
||||
|
||||
project = await projects_svc.get_project(user_id, project_id)
|
||||
if project is None:
|
||||
raise ValueError(f"project {project_id} not found")
|
||||
designs = await design_systems_svc.list_design_systems(user_id)
|
||||
systems = await systems_svc.list_systems(user_id, project_id, include_archived=True)
|
||||
catalog = await platforms_svc.list_platforms()
|
||||
return {
|
||||
"design_system_id": project.design_system_id,
|
||||
"design_systems": [{"id": d.id, "title": d.title} for d in designs],
|
||||
"systems": len(systems),
|
||||
"platforms": [{"slug": p.slug, "name": p.name} for p in catalog],
|
||||
"project_platforms": await platforms_svc.project_platforms(user_id, project_id) or [],
|
||||
}
|
||||
|
||||
|
||||
async def _check_targets(user_id: int, choices: dict) -> None:
|
||||
"""Every id a decision names must be the caller's (or readable) BEFORE any
|
||||
effect lands — a decision applies whole or errors whole."""
|
||||
from grimoire.services import access
|
||||
|
||||
from grimoire.services import platforms as platforms_svc
|
||||
|
||||
ds = choices["design_system_id"]
|
||||
if ds is not None and not await access.can_read_design_system(user_id, ds):
|
||||
raise ValueError(f"design system {ds} not found (or not readable)")
|
||||
if choices["platforms"]:
|
||||
_, unknown = await platforms_svc.resolve_slugs(choices["platforms"])
|
||||
if unknown:
|
||||
raise ValueError(f"unknown platform(s): {', '.join(unknown)} (list_platforms)")
|
||||
|
||||
|
||||
async def decide(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
*,
|
||||
choices: dict | None,
|
||||
via: str,
|
||||
) -> dict:
|
||||
"""Record a project's inception decision and apply it.
|
||||
|
||||
Owner-only. Validates the choices (pure) and every target (readable)
|
||||
first; then, each idempotent: point the project at the design system
|
||||
(None = explicitly none), seed the standard Systems if asked and the
|
||||
project has none; then write ``projects.inception`` LAST. Re-deciding
|
||||
replaces the design system and re-seeds nothing a project already has.
|
||||
|
||||
Returns {"inception": <record>, "effects": {design_system_id,
|
||||
systems_seeded, platforms}} — `platforms` is the project's answers after
|
||||
the decision, or None when the choice was left unstated.
|
||||
"""
|
||||
from grimoire.services import design_systems as design_systems_svc
|
||||
from grimoire.services import projects as projects_svc
|
||||
from grimoire.services import systems as systems_svc
|
||||
|
||||
if via not in INCEPTION_VIAS or via == "legacy":
|
||||
raise ValueError("via must be 'mcp' or 'ui' ('legacy' is the migration's stamp)")
|
||||
error = validate_inception(choices or {})
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
choices = normalize_choices(choices)
|
||||
project = await projects_svc.get_project(user_id, project_id) # owner-scoped
|
||||
if project is None:
|
||||
raise ValueError(f"project {project_id} not found (or not yours)")
|
||||
await _check_targets(user_id, choices)
|
||||
|
||||
if not await design_systems_svc.set_project_design_system(
|
||||
user_id, project_id, choices["design_system_id"]
|
||||
):
|
||||
raise ValueError("could not set the design system (no write on the project?)")
|
||||
seeded = (
|
||||
await systems_svc.seed_standard_systems(user_id, project_id)
|
||||
if choices["seed_systems"] else []
|
||||
)
|
||||
platforms = None
|
||||
if choices["platforms"] is not None:
|
||||
platforms = await _apply_platforms(user_id, project_id, choices["platforms"])
|
||||
|
||||
record = {
|
||||
"decided_at": datetime.now(timezone.utc).isoformat(),
|
||||
"decided_by": user_id,
|
||||
"via": via,
|
||||
"choices": choices,
|
||||
}
|
||||
async with async_session() as session:
|
||||
row = await session.get(Project, project_id)
|
||||
row.inception = record
|
||||
row.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return {
|
||||
"inception": record,
|
||||
"effects": {
|
||||
"design_system_id": choices["design_system_id"],
|
||||
"systems_seeded": [sy.name for sy in seeded],
|
||||
"platforms": platforms,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _apply_platforms(user_id: int, project_id: int, slugs: list[str]) -> list[dict]:
|
||||
"""The platforms answer as membership. The list is the WHOLE answer:
|
||||
every slug in it is declared, and every platform the project was a member
|
||||
of (declared or detected) that is left out becomes rejected — so the next
|
||||
refresh cannot detect back what the person just said the project isn't.
|
||||
Platforms already rejected stay rejected; platforms never answered stay
|
||||
unanswered."""
|
||||
from grimoire.services import platforms as platforms_svc
|
||||
|
||||
named = set(slugs)
|
||||
current = await platforms_svc.project_platforms(user_id, project_id) or []
|
||||
updates: dict[str, str | None] = {slug: "declared" for slug in named}
|
||||
for row in current:
|
||||
if row["slug"] not in named and row["state"] in platforms_svc.MEMBER_STATES:
|
||||
updates[row["slug"]] = "rejected"
|
||||
return await platforms_svc.set_project_platforms(user_id, project_id, updates)
|
||||
|
||||
|
||||
def _platform_line(defaults: dict) -> str:
|
||||
"""What the ask says about platforms: what detection already found, and
|
||||
the catalog to choose from."""
|
||||
found = [
|
||||
p["slug"] for p in defaults.get("project_platforms", [])
|
||||
if p["state"] in ("declared", "detected")
|
||||
]
|
||||
catalog = ", ".join(p["slug"] for p in defaults.get("platforms", [])) or "none"
|
||||
lead = f"detected so far: {', '.join(found)}; " if found else ""
|
||||
return f"{lead}catalog: {catalog}"
|
||||
|
||||
|
||||
async def inception_ask(user_id: int, project_id: int) -> dict:
|
||||
"""The enter_project ask for an undecided project — the sibling of the
|
||||
systems-bootstrap ask: the project's OWN current
|
||||
defaults, what to ask the operator, and the exact call that answers it.
|
||||
Fail-open: a hint must never break the call it rides on."""
|
||||
try:
|
||||
defaults = await current_defaults(user_id, project_id)
|
||||
except Exception:
|
||||
return {}
|
||||
designs = ", ".join(f"{d['title']} (#{d['id']})" for d in defaults["design_systems"]) or "none"
|
||||
return {
|
||||
"defaults": defaults,
|
||||
"ask": (
|
||||
"This project has no inception decision: nobody has said what it "
|
||||
"inherits. Design system — "
|
||||
f"{'#' + str(defaults['design_system_id']) if defaults['design_system_id'] else 'none'} "
|
||||
f"(available: {designs}); Systems — {defaults['systems']}. Ask the operator, "
|
||||
"once: which design system (or none), whether to seed "
|
||||
"the standard starter Systems, and which platforms the project is "
|
||||
f"built on or ships as ({_platform_line(defaults)}) — then record the "
|
||||
"answers. This ask repeats on every enter_project until a decision "
|
||||
"is recorded."
|
||||
),
|
||||
"call": (
|
||||
f"decide_project_inception(project_id={project_id}, "
|
||||
"design_system_id=<id | -1 for none>, seed_systems=<true|false>, "
|
||||
"platforms=[<slug>, ...])"
|
||||
),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,782 @@
|
||||
"""Knowledge service — one query across every record kind Grimoire holds.
|
||||
|
||||
ACL (services/access.py): these queries honour shares, so a record shared with
|
||||
you can be *found* as well as opened by id — at two different widths:
|
||||
|
||||
- **searching** (a `q` the caller typed) uses the full read scope, so a record
|
||||
shared directly with you is findable when you go looking for it. BOTH halves
|
||||
of the hybrid search — keyword and semantic — see equally, or a record would
|
||||
be findable by wording and invisible by meaning;
|
||||
- **browsing** (no `q`) and the facet counts beside it use the narrower browse
|
||||
scope: your own records plus anything in a project you can reach.
|
||||
|
||||
The asymmetry is the point. A record that appears unasked reads as one you
|
||||
endorsed, so a one-off direct share has to be searched for rather than arriving
|
||||
in your ambient lists.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import logging
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.services.access import browsable_notes_clause, readable_notes_clause
|
||||
from grimoire.services.text import excerpt_fields
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SNIPPET_LEN = 200
|
||||
|
||||
|
||||
# --- the location filter (reverse lookup) ------------------------------------
|
||||
#
|
||||
# ONE predicate in two dialects: "this record carries a location matching every
|
||||
# part asked for." The SQL form runs in the browse arm and the keyword arm,
|
||||
# before the count and the page slice, so totals stay honest; the Python form
|
||||
# runs in the semantic arm, which post-filters candidates it already holds in
|
||||
# memory. THE TWO MUST CHANGE TOGETHER — a filter only one arm applies makes a
|
||||
# snippet findable by wording and invisible by location, which is exactly the
|
||||
# split tests/test_retrieval_scopes.py exists to prevent.
|
||||
#
|
||||
# Semantics, both dialects:
|
||||
# - parts are ANDed WITHIN a single location entry, never across the list. A
|
||||
# snippet whose first location is in repo A and whose second is at path B
|
||||
# does not answer repo=A + path=B — it was never in that place.
|
||||
# - `path` matches exactly OR as a directory prefix: "frontend/src" finds
|
||||
# "frontend/src/lib/x.ts". Prefix is the one part a GIN containment lookup
|
||||
# can't serve (migration 0070), which is why this is a jsonpath `@?` rather
|
||||
# than `@>` — jsonpath `starts with` is index-served by the same GIN index.
|
||||
#
|
||||
# The filter reads `notes.data`, so it only sees rows carrying that mirror.
|
||||
# Rows written before migration 0070 are populated once at startup by
|
||||
# snippets.backfill_snippet_data — without that, this query would answer "no
|
||||
# snippets here" for an old snippet and the caller would write the helper again.
|
||||
|
||||
LOCATION_KEYS = ("repo", "path", "symbol")
|
||||
|
||||
|
||||
def location_parts(repo: str = "", path: str = "", symbol: str = "") -> dict[str, str]:
|
||||
"""The non-empty, stripped location parts asked for; {} when none were."""
|
||||
given = {"repo": repo, "path": path, "symbol": symbol}
|
||||
return {k: (v or "").strip() for k, v in given.items() if (v or "").strip()}
|
||||
|
||||
|
||||
def _path_matches(have: str, want: str) -> bool:
|
||||
"""Exact, or `have` sits somewhere under the `want` directory."""
|
||||
return have == want or have.startswith(want.rstrip("/") + "/")
|
||||
|
||||
|
||||
def location_matches(data: dict | None, parts: dict[str, str]) -> bool:
|
||||
"""Python dialect of the location predicate. Keep in step with _location_clause."""
|
||||
if not parts:
|
||||
return True
|
||||
for loc in (data or {}).get("locations") or []:
|
||||
if all(
|
||||
_path_matches((loc.get(key) or "").strip(), want)
|
||||
if key == "path"
|
||||
# Repo names are recorded free-form ("Grimoire" / "FabledScribe" /
|
||||
# "fabledscribe") — case is never the distinguishing thing.
|
||||
else (loc.get(key) or "").strip().lower() == want.lower()
|
||||
if key == "repo"
|
||||
else (loc.get(key) or "").strip() == want
|
||||
for key, want in parts.items()
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def location_jsonpath(parts: dict[str, str]) -> str:
|
||||
"""The jsonpath behind the SQL dialect — one `locations` entry matching all parts.
|
||||
|
||||
Values are embedded as JSON string literals (jsonpath uses JSON quoting), so
|
||||
a repo or path carrying a quote can't break out of the expression. The keys
|
||||
are our own fixed set, never caller input.
|
||||
"""
|
||||
filters = []
|
||||
for key in LOCATION_KEYS:
|
||||
if key not in parts:
|
||||
continue
|
||||
want = parts[key]
|
||||
literal = json.dumps(want)
|
||||
if key == "path":
|
||||
prefix = json.dumps(want.rstrip("/") + "/")
|
||||
filters.append(f"(@.path == {literal} || @.path starts with {prefix})")
|
||||
elif key == "repo":
|
||||
# Case-insensitive, anchored, regex-escaped — mirrors the
|
||||
# Python dialect's .lower() compare.
|
||||
pattern = json.dumps("^" + re.escape(want) + "$")
|
||||
filters.append(f'(@.repo like_regex {pattern} flag "i")')
|
||||
else:
|
||||
filters.append(f"@.{key} == {literal}")
|
||||
return f"$.locations[*] ? ({' && '.join(filters)})"
|
||||
|
||||
|
||||
def _location_clause(parts: dict[str, str]):
|
||||
"""SQL dialect of the location predicate. Keep in step with location_matches."""
|
||||
return Note.data.path_exists(location_jsonpath(parts))
|
||||
|
||||
|
||||
# --- drift-check filter ------------------------------------------------------
|
||||
# `verification` selects on the drift-check verdict stored in `data.verification`
|
||||
# (see services/snippets.py). Statuses are the service's own constants; the two
|
||||
# composite values are what the operator actually asks for.
|
||||
#
|
||||
# The interesting one is `attention`. A verdict describes the code it was checked
|
||||
# against, so an OK verdict on code that has since been edited is not an OK
|
||||
# record — nobody has checked what's actually there. That is expressible in SQL
|
||||
# only because `data.code_sha` mirrors the current code's fingerprint alongside
|
||||
# the verdict's: jsonpath compares the two fields within the row, so this stays
|
||||
# one index-served predicate rather than a post-filter that would make the
|
||||
# pagination total a lie.
|
||||
#
|
||||
# Same two-dialect discipline as the location filter above, and the same hazard:
|
||||
# THE TWO MUST CHANGE TOGETHER. tests/test_snippet_drift_check.py is the guard —
|
||||
# it walks both dialects over the same cases, including the one that motivated
|
||||
# `attention` existing (an ok verdict whose code_sha has gone stale, which is
|
||||
# neither `drifted` nor `unverified` yet plainly needs looking at).
|
||||
|
||||
def verification_matches(data: dict | None, value: str) -> bool:
|
||||
"""Python dialect of the verification predicate.
|
||||
|
||||
Keep in step with _verification_clause — the semantic arm's candidates are
|
||||
already fetched, so there is no query left to narrow and the same rule has
|
||||
to be expressible twice. Same arrangement as location_matches.
|
||||
"""
|
||||
want = (value or "").strip().lower()
|
||||
if not want:
|
||||
return True
|
||||
verdict = (data or {}).get("verification") or {}
|
||||
status = verdict.get("status") or ""
|
||||
if not status:
|
||||
return want == "unverified"
|
||||
expired = verdict.get("code_sha") != (data or {}).get("code_sha")
|
||||
# A push touched the recorded location since the verdict: the repo
|
||||
# moved under it, so it needs a look even though it hasn't failed.
|
||||
invalidated = bool(verdict.get("invalidated_by"))
|
||||
if want == "unverified":
|
||||
return False
|
||||
if want == "drifted":
|
||||
return status != "ok"
|
||||
if want == "attention":
|
||||
return status != "ok" or expired or invalidated
|
||||
if want == "ok":
|
||||
return status == "ok" and not expired and not invalidated
|
||||
return status == want
|
||||
|
||||
|
||||
_VERIFY_DRIFTED_JSONPATH = '$.verification ? (@.status != "ok")'
|
||||
_VERIFY_EXPIRED_JSONPATH = "$ ? (@.verification.code_sha != @.code_sha)"
|
||||
_VERIFY_ANY_JSONPATH = "$.verification"
|
||||
# A push touched the recorded location since the verdict.
|
||||
_VERIFY_INVALIDATED_JSONPATH = "$.verification.invalidated_by"
|
||||
|
||||
|
||||
def _verification_clause(value: str):
|
||||
"""SQL predicate for one `verification` filter value, or None for no filter."""
|
||||
want = (value or "").strip().lower()
|
||||
if not want:
|
||||
return None
|
||||
has_verdict = Note.data.path_exists(_VERIFY_ANY_JSONPATH)
|
||||
if want == "unverified":
|
||||
# Never checked at all. Rows predating migration 0070 have no `data`
|
||||
# whatsoever and land here correctly — which is right, they haven't been.
|
||||
return ~has_verdict
|
||||
if want == "drifted":
|
||||
return Note.data.path_exists(_VERIFY_DRIFTED_JSONPATH)
|
||||
if want == "attention":
|
||||
# Everything worth looking at: a failing verdict, an expired one, OR
|
||||
# one whose recorded location a push has since touched.
|
||||
return or_(
|
||||
Note.data.path_exists(_VERIFY_DRIFTED_JSONPATH),
|
||||
and_(has_verdict, Note.data.path_exists(_VERIFY_EXPIRED_JSONPATH)),
|
||||
Note.data.path_exists(_VERIFY_INVALIDATED_JSONPATH),
|
||||
)
|
||||
if want == "ok":
|
||||
# A clean bill of health that still describes the current code. The
|
||||
# `~expired` half matters: without it this would quietly include records
|
||||
# whose blessing has lapsed, which is the exact failure the feature is
|
||||
# meant to catch. Same for push-invalidation — "ok" must mean the repo
|
||||
# hasn't moved under the verdict either.
|
||||
return and_(
|
||||
Note.data.path_exists('$.verification ? (@.status == "ok")'),
|
||||
~Note.data.path_exists(_VERIFY_EXPIRED_JSONPATH),
|
||||
~Note.data.path_exists(_VERIFY_INVALIDATED_JSONPATH),
|
||||
)
|
||||
# A specific status: 'missing' | 'moved' | 'changed'.
|
||||
return Note.data.path_exists(
|
||||
f"$.verification ? (@.status == {json.dumps(want)})"
|
||||
)
|
||||
|
||||
|
||||
def _note_to_item(note: Note, chunks: dict[int, dict] | None = None) -> dict:
|
||||
"""One row for a browse or search listing.
|
||||
|
||||
`chunks` is `report["best_chunk"]` from a semantic search, when this row
|
||||
came from one. With it the card shows the passage that MATCHED; without it
|
||||
— a plain listing, where nothing was matched and so no span is better than
|
||||
any other — it shows the record's opening. `snippet_is` says which, on
|
||||
every row, so the two never have to be told apart by guessing.
|
||||
|
||||
The head of the body on every row alike (`body[:_SNIPPET_LEN]`) would show
|
||||
a record matched on its sixth paragraph its first, which the search had
|
||||
already scored lower, with nothing marking the cut.
|
||||
"""
|
||||
item: dict = {
|
||||
"id": note.id,
|
||||
"note_type": note.note_type or "note",
|
||||
"title": note.title,
|
||||
**excerpt_fields(
|
||||
note.body or "",
|
||||
(chunks or {}).get(int(note.id)),
|
||||
_SNIPPET_LEN,
|
||||
key="snippet",
|
||||
),
|
||||
"tags": note.tags or [],
|
||||
"project_id": note.project_id,
|
||||
# These lists now include records shared with the caller, so the client
|
||||
# needs the owner to tell "mine" from "someone else's" in a mixed list.
|
||||
"user_id": note.user_id,
|
||||
"created_at": iso(note.created_at),
|
||||
"updated_at": iso(note.updated_at),
|
||||
}
|
||||
# Drift verdict, when one has been recorded. Included here rather
|
||||
# than decorated on by the snippet layer because `current` is derivable from
|
||||
# `data` alone — the verdict's code_sha against the row's — so this needs no
|
||||
# body parsing and stays a plain projection of the column. Omitted entirely
|
||||
# when unchecked, so "no key" and "never verified" don't become two states
|
||||
# the client has to tell apart.
|
||||
# Snippet language, same reasoning as the verdict below: a plain projection of
|
||||
# the `data` mirror, no body parsing. Needed because a prior-art hit in a
|
||||
# DIFFERENT language than the file being written is useful as the shape of a
|
||||
# solution but must not be mistaken for code to paste — and the caller
|
||||
# can only say "different" if the language is on the item.
|
||||
language = (note.data or {}).get("language") if note.data else None
|
||||
if language:
|
||||
item["language"] = language
|
||||
|
||||
# WHEN a lesson applies, same reasoning as the language above: a plain
|
||||
# projection of the `data` mirror, no body parsing. A listing of lessons
|
||||
# without it is a list of claims with the situation — the half that says
|
||||
# when each one matters — left off, and the title's own copy cannot be
|
||||
# recovered by splitting on the em dash, because a claim may contain one.
|
||||
trigger = (note.data or {}).get("when_to_apply") if note.data else None
|
||||
if trigger:
|
||||
item["when_to_apply"] = trigger
|
||||
# A snippet's, for the same reason — the stored title does not carry it,
|
||||
# so without this a list shows names with no situation.
|
||||
usage = (note.data or {}).get("when_to_use") if note.data else None
|
||||
if usage:
|
||||
item["when_to_use"] = usage
|
||||
|
||||
verdict = (note.data or {}).get("verification") if note.data else None
|
||||
if verdict and verdict.get("status"):
|
||||
item["verification"] = {
|
||||
"status": verdict["status"],
|
||||
"current": verdict.get("code_sha") == (note.data or {}).get("code_sha"),
|
||||
"checked_at": verdict.get("checked_at"),
|
||||
"detail": verdict.get("detail"),
|
||||
"path": verdict.get("path"),
|
||||
}
|
||||
# Present only when a push has touched the recorded location since the
|
||||
# verdict — the "recheck me" marker, cleared by re-verifying.
|
||||
if verdict.get("invalidated_by"):
|
||||
item["verification"]["invalidated_by"] = verdict["invalidated_by"]
|
||||
|
||||
# Task fields — override note_type and add status/priority/due_date
|
||||
if note.is_task:
|
||||
item["note_type"] = "task"
|
||||
item["task_kind"] = note.task_kind
|
||||
item["status"] = note.status
|
||||
item["priority"] = note.priority
|
||||
item["due_date"] = iso(note.due_date)
|
||||
|
||||
return item
|
||||
|
||||
|
||||
# What each type facet MEANS, once, for every arm that has to know.
|
||||
#
|
||||
# The vocabulary spans BOTH typing axes — `note_type` for non-task records and
|
||||
# `task_kind` for tasks — so a facet cannot be a filter on one column, which is
|
||||
# why this is a table rather than a chain of ifs. Each entry is
|
||||
# (is_task, the value pinned on that axis); None pins nothing, i.e. every task.
|
||||
#
|
||||
# It is a table because the predicate has three dialects — SQL, a Python test
|
||||
# over semantic candidates, and the `is_task` pre-filter — and three hand-kept
|
||||
# copies agree only by luck. Adding `issue` to the SQL arm alone would set the
|
||||
# pre-filter to is_task=False, hand the Python arm a candidate set containing
|
||||
# no tasks at all, and return an empty semantic half for the Issues facet
|
||||
# forever, with nothing red anywhere. A new facet is one row here.
|
||||
#
|
||||
# `plan` is retired but kept: legacy plan-tasks may exist, and a facet they
|
||||
# answer to costs one line. It has no chip in the UI.
|
||||
_FACETS: dict[str, tuple[bool, str | None]] = {
|
||||
"task": (True, None),
|
||||
"work": (True, "work"),
|
||||
"issue": (True, "issue"),
|
||||
"spike": (True, "spike"),
|
||||
"plan": (True, "plan"),
|
||||
"note": (False, "note"),
|
||||
"process": (False, "process"),
|
||||
"snippet": (False, "snippet"),
|
||||
# A transferable insight, keyed by the situation it applies to rather than
|
||||
# by its topic. It reaches the browse surface through this one entry: the
|
||||
# door's validation, the counts and both dialects of the type filter are
|
||||
# all generated from this table, so adding a kind is a single edit rather
|
||||
# than four that must agree.
|
||||
"lesson": (False, "lesson"),
|
||||
}
|
||||
|
||||
# The non-task record types, for the counts query. Derived so it cannot drift
|
||||
# from the table above.
|
||||
NON_TASK_FACETS = tuple(
|
||||
value for _is_task, value in _FACETS.values() if not _is_task and value
|
||||
)
|
||||
|
||||
# The whole vocabulary, for the door's request validation — public so the route
|
||||
# validates against the same table the query reads instead of a hand-kept copy.
|
||||
FACET_TYPES = frozenset(_FACETS)
|
||||
|
||||
|
||||
# An unrecognised facet resolves to "a non-task note whose note_type is that
|
||||
# string" — which matches nothing, since no row stores an unknown type. That is
|
||||
# the behaviour the old if-chain had by falling through, and it is the right
|
||||
# one: a typo should return an empty list, never the whole corpus.
|
||||
def _facet(note_type: str) -> tuple[bool, str | None]:
|
||||
return _FACETS.get(note_type, (False, note_type))
|
||||
|
||||
|
||||
def facet_is_task(note_type: str | None) -> bool | None:
|
||||
"""The `is_task` pre-filter a facet implies — None when it spans both.
|
||||
|
||||
Used to narrow the semantic candidate set before it is fetched. Reads the
|
||||
same table `_apply_type_filter` and `matches_facet` read, so the pre-filter
|
||||
cannot disagree with the predicate it is meant to anticipate.
|
||||
"""
|
||||
if not note_type:
|
||||
return None
|
||||
return _facet(note_type)[0]
|
||||
|
||||
|
||||
def matches_facet(note, note_type: str | None) -> bool:
|
||||
"""The Python dialect of `_apply_type_filter`, for candidates the vector
|
||||
search has already fetched — there is no query left to narrow.
|
||||
|
||||
Generated from the same table, so this is a translation rather than a
|
||||
second implementation. Note the `not note.is_task` arm: the hand-written
|
||||
version omitted it and was saved only by the upstream pre-filter.
|
||||
"""
|
||||
if not note_type:
|
||||
return True
|
||||
is_task, value = _facet(note_type)
|
||||
if is_task:
|
||||
return note.is_task and (value is None or note.task_kind == value)
|
||||
return not note.is_task and note.note_type == value
|
||||
|
||||
|
||||
def search_filters_for(facet: str) -> dict:
|
||||
"""The semantic-search kwargs one facet implies: is_task + note_type/task_kind.
|
||||
|
||||
A third dialect of the same table, for the arm that has no SQL statement to
|
||||
narrow and no fetched row to test — it is passing filters INTO
|
||||
`semantic_search_notes`. `_apply_type_filter` is the SQL dialect and
|
||||
`matches_facet` the Python one; all three read `_FACETS`, which is what
|
||||
keeps "adding a kind" a single edit.
|
||||
|
||||
Returns kwargs rather than a tuple so a caller splats it and cannot pair
|
||||
`task_kind` with `is_task=False` by writing the positions out of order.
|
||||
"""
|
||||
is_task, value = _facet(facet)
|
||||
if is_task:
|
||||
return {"is_task": True, "task_kind": value}
|
||||
return {"is_task": False, "note_type": value}
|
||||
|
||||
|
||||
# The two names a `content_type` parameter carries that are not facets. Both
|
||||
# search doors that take one — the MCP tool and /api/search — have always
|
||||
# spelled them this way, so they are the contract rather than a convenience:
|
||||
#
|
||||
# 'all' (and empty) — no kind filter at all.
|
||||
# 'note' — ANY non-task record, so snippets, lessons and processes
|
||||
# are all still in scope. The BROWSE facet of the same
|
||||
# name is narrower (`note_type == 'note'` exactly). The
|
||||
# two are deliberately different: narrowing this one
|
||||
# would stop returning snippets to every caller that
|
||||
# already asks this way, and the doors document the
|
||||
# containment instead.
|
||||
_CONTENT_TYPE_BROAD = {
|
||||
"": {"is_task": None},
|
||||
"all": {"is_task": None},
|
||||
"note": {"is_task": False},
|
||||
}
|
||||
|
||||
|
||||
def content_type_filters(content_type: str, extra: tuple = ()) -> dict:
|
||||
"""The search kwargs a door's `content_type` parameter implies.
|
||||
|
||||
Lives here, beside `_FACETS`, because a door that keeps its own map drifts:
|
||||
one door offers two kinds while another offers nine. Derived, so adding a
|
||||
kind stays one edit.
|
||||
|
||||
Raises on anything unrecognised. `_facet` alone would fall back to reading
|
||||
an unknown string as a note_type, which matches no row — so a typo comes
|
||||
back as a confident empty result, and an empty result is a CLAIM that the
|
||||
corpus holds nothing of the sort. A door has to be able to say "that is not
|
||||
a kind" instead. `extra` names kinds the caller dispatches elsewhere (the
|
||||
MCP tool's 'rule' and 'milestone'), so the refusal lists what that door
|
||||
really accepts and not a vocabulary from some other door.
|
||||
"""
|
||||
if content_type in _CONTENT_TYPE_BROAD:
|
||||
return dict(_CONTENT_TYPE_BROAD[content_type])
|
||||
if content_type in FACET_TYPES:
|
||||
return search_filters_for(content_type)
|
||||
raise ValueError(
|
||||
f"unknown content_type {content_type!r}. Valid: "
|
||||
+ ", ".join(sorted({"all", *FACET_TYPES, *extra}))
|
||||
)
|
||||
|
||||
|
||||
def _apply_type_filter(stmt, note_type: str | None):
|
||||
"""Apply the type facet to a Note select. Trashed rows are always excluded."""
|
||||
stmt = stmt.where(Note.deleted_at.is_(None))
|
||||
if not note_type:
|
||||
return stmt
|
||||
is_task, value = _facet(note_type)
|
||||
if is_task:
|
||||
stmt = stmt.where(Note.status.isnot(None))
|
||||
if value is not None:
|
||||
stmt = stmt.where(Note.task_kind == value)
|
||||
return stmt
|
||||
return stmt.where(Note.status.is_(None)).where(Note.note_type == value)
|
||||
|
||||
|
||||
async def query_knowledge(
|
||||
user_id: int,
|
||||
note_type: str | None,
|
||||
tags: list[str],
|
||||
sort: str,
|
||||
q: str | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
project_id: int | None = None,
|
||||
locations: dict[str, str] | None = None,
|
||||
verification: str = "",
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Query knowledge objects with filters.
|
||||
|
||||
`project_id` narrows to one project (None = every project).
|
||||
|
||||
`locations` narrows to records whose `data.locations` holds an entry matching
|
||||
every part given — build it with `location_parts(repo=…, path=…, symbol=…)`.
|
||||
Today only snippets carry locations, but the column is general, so the filter
|
||||
lives here with the query rather than in one type's service.
|
||||
|
||||
`verification` narrows on the drift-check verdict: 'ok', 'drifted',
|
||||
'unverified', 'attention', or one specific failure ('missing' | 'moved' |
|
||||
'changed'). Empty means no filter.
|
||||
|
||||
Returns (items, total_count).
|
||||
"""
|
||||
# Semantic search path — scores take priority over sort
|
||||
if q:
|
||||
return await _semantic_knowledge_search(
|
||||
user_id, q, note_type=note_type, tags=tags, limit=limit,
|
||||
offset=offset, project_id=project_id, locations=locations,
|
||||
verification=verification,
|
||||
)
|
||||
|
||||
# No query = browsing. Narrower scope: a record shared directly with the
|
||||
# caller is search-only and must not appear in an ambient list.
|
||||
visible = browsable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
base = select(Note).where(visible)
|
||||
|
||||
base = _apply_type_filter(base, note_type)
|
||||
if project_id is not None:
|
||||
base = base.where(Note.project_id == project_id)
|
||||
|
||||
for tag in tags:
|
||||
base = base.where(Note.tags.contains([tag]))
|
||||
|
||||
if locations:
|
||||
base = base.where(_location_clause(locations))
|
||||
|
||||
verify_clause = _verification_clause(verification)
|
||||
if verify_clause is not None:
|
||||
base = base.where(verify_clause)
|
||||
|
||||
# Count before pagination
|
||||
count_stmt = select(func.count()).select_from(base.subquery())
|
||||
total: int = (await session.execute(count_stmt)).scalar_one()
|
||||
|
||||
# Apply sort
|
||||
if sort == "created":
|
||||
base = base.order_by(Note.created_at.desc())
|
||||
elif sort == "alpha":
|
||||
base = base.order_by(Note.title.asc())
|
||||
elif sort == "type":
|
||||
base = base.order_by(Note.note_type.asc(), Note.updated_at.desc())
|
||||
else: # modified (default)
|
||||
base = base.order_by(Note.updated_at.desc())
|
||||
|
||||
rows = list((await session.execute(base.limit(limit).offset(offset))).scalars().all())
|
||||
|
||||
return [_note_to_item(n) for n in rows], total
|
||||
|
||||
|
||||
async def _semantic_knowledge_search(
|
||||
user_id: int,
|
||||
q: str,
|
||||
note_type: str | None,
|
||||
tags: list[str],
|
||||
limit: int,
|
||||
offset: int,
|
||||
project_id: int | None = None,
|
||||
locations: dict[str, str] | None = None,
|
||||
verification: str = "",
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Hybrid search: keyword matches first (title/body ILIKE), then semantic results.
|
||||
|
||||
Exact keyword matches always rank above semantic-only matches so that
|
||||
searching for a name like "Weston" surfaces the note with that title
|
||||
before conceptually related notes.
|
||||
|
||||
BEST-EFFORT TOP-N, not exhaustive pagination: the ranked candidate set is
|
||||
capped (keyword limit*2 + up to ~200 semantic), so `total` is the size of
|
||||
that window, NOT the true match count, and matches beyond the cap are not
|
||||
reachable by paging. Each page also recomputes the full merge (O(corpus)
|
||||
per page). Acceptable for an interactive "best results" feed; a cached
|
||||
ranked-id list or pgvector ORDER BY/LIMIT is the fix if exhaustive,
|
||||
cheap pagination is ever needed.
|
||||
"""
|
||||
# 1. Keyword search — title and body ILIKE
|
||||
keyword_notes: list[Note] = []
|
||||
try:
|
||||
# A typed query is an explicit act, so it reaches the caller's full read
|
||||
# scope — including records shared directly with them.
|
||||
visible = readable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
pattern = f"%{q}%"
|
||||
base = (
|
||||
select(Note)
|
||||
.where(visible)
|
||||
.where(Note.title.ilike(pattern) | Note.body.ilike(pattern))
|
||||
)
|
||||
base = _apply_type_filter(base, note_type)
|
||||
if project_id is not None:
|
||||
base = base.where(Note.project_id == project_id)
|
||||
for tag in tags:
|
||||
base = base.where(Note.tags.contains([tag]))
|
||||
if locations:
|
||||
base = base.where(_location_clause(locations))
|
||||
verify_clause = _verification_clause(verification)
|
||||
if verify_clause is not None:
|
||||
base = base.where(verify_clause)
|
||||
# Title matches first, then body-only matches, newest first within each
|
||||
base = base.order_by(
|
||||
Note.title.ilike(pattern).desc(),
|
||||
Note.updated_at.desc(),
|
||||
).limit(limit * 2)
|
||||
keyword_notes = list((await session.execute(base)).scalars().all())
|
||||
except Exception:
|
||||
logger.warning("Keyword search failed", exc_info=True)
|
||||
|
||||
# 2. Semantic search — conceptual similarity, at the SAME scope as the
|
||||
# keyword half above. Both halves of one search must see equally, or a shared
|
||||
# record would be findable by wording and invisible by meaning — which is the
|
||||
# case a semantic search exists to serve.
|
||||
semantic_notes: list[Note] = []
|
||||
# Filled by the search below; stays empty when the embedder is down or the
|
||||
# call raises, in which case every row falls back to its opening.
|
||||
_semantic_report: dict = {}
|
||||
try:
|
||||
from grimoire.services.embeddings import (
|
||||
INTERACTIVE_SEARCH_THRESHOLD,
|
||||
semantic_search_notes,
|
||||
)
|
||||
is_task_filter = facet_is_task(note_type)
|
||||
import time as _time
|
||||
_t0 = _time.perf_counter()
|
||||
candidates = await semantic_search_notes(
|
||||
user_id=user_id,
|
||||
scope="read",
|
||||
query=q,
|
||||
report=_semantic_report,
|
||||
limit=min(200, limit * 4),
|
||||
# The shared interactive floor, one constant, so no copy of the
|
||||
# number can move without the others.
|
||||
threshold=INTERACTIVE_SEARCH_THRESHOLD,
|
||||
is_task=is_task_filter,
|
||||
project_id=project_id,
|
||||
)
|
||||
# The human's MAIN search surface, logged under its own source so
|
||||
# threshold tuning can include or exclude human queries deliberately
|
||||
# rather than by accident.
|
||||
from grimoire.services.retrieval_telemetry import record_retrieval
|
||||
record_retrieval(
|
||||
user_id=user_id, source="browse_search", query=q,
|
||||
threshold=INTERACTIVE_SEARCH_THRESHOLD, limit=min(200, limit * 4),
|
||||
project_id=project_id, is_task=is_task_filter, results=candidates,
|
||||
duration_ms=(_time.perf_counter() - _t0) * 1000.0,
|
||||
)
|
||||
for _score, note in candidates:
|
||||
if note.deleted_at is not None:
|
||||
continue
|
||||
if not matches_facet(note, note_type):
|
||||
continue
|
||||
if tags and not all(t in (note.tags or []) for t in tags):
|
||||
continue
|
||||
# The Python dialect of the same predicate the SQL arms apply above —
|
||||
# these candidates arrive already fetched, so there's no query to
|
||||
# narrow. See the comment on location_matches.
|
||||
if locations and not location_matches(note.data, locations):
|
||||
continue
|
||||
if verification and not verification_matches(note.data, verification):
|
||||
continue
|
||||
semantic_notes.append(note)
|
||||
except Exception:
|
||||
logger.warning("Semantic search unavailable, using keyword results only", exc_info=True)
|
||||
|
||||
# 3. Merge — keyword matches first, then semantic (deduplicated)
|
||||
seen_ids: set[int] = set()
|
||||
merged: list[Note] = []
|
||||
for note in keyword_notes:
|
||||
if note.id not in seen_ids:
|
||||
seen_ids.add(note.id)
|
||||
merged.append(note)
|
||||
for note in semantic_notes:
|
||||
if note.id not in seen_ids:
|
||||
seen_ids.add(note.id)
|
||||
merged.append(note)
|
||||
|
||||
total = len(merged)
|
||||
page_items = merged[offset: offset + limit]
|
||||
return [
|
||||
_note_to_item(n, _semantic_report.get("best_chunk"))
|
||||
for n in page_items
|
||||
], total
|
||||
|
||||
|
||||
async def get_knowledge_tags(user_id: int, note_type: str | None = None) -> list[str]:
|
||||
"""Distinct tags across what this user can BROWSE.
|
||||
|
||||
Follows the browse list rather than the read scope: a facet is itself a
|
||||
passive surface, and offering a tag that only a search-only record carries
|
||||
would filter the visible list down to nothing."""
|
||||
visible = browsable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
base = (
|
||||
select(func.unnest(Note.tags).label("tag"))
|
||||
.where(visible)
|
||||
)
|
||||
base = _apply_type_filter(base, note_type)
|
||||
stmt = base.distinct().order_by("tag")
|
||||
rows = list((await session.execute(stmt)).scalars().all())
|
||||
return [r for r in rows if r]
|
||||
|
||||
|
||||
async def get_knowledge_counts(user_id: int, tags: list[str] | None = None) -> dict[str, int]:
|
||||
"""Per-type counts for the sidebar, over what this user can BROWSE — so the
|
||||
numbers match the list they sit beside rather than promising rows that only a
|
||||
search would surface."""
|
||||
visible = browsable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
def _scoped(stmt):
|
||||
stmt = stmt.where(visible).where(Note.deleted_at.is_(None))
|
||||
for tag in tags or []:
|
||||
stmt = stmt.where(Note.tags.contains([tag]))
|
||||
return stmt
|
||||
|
||||
# One grouped query per typing axis. A count per kind needs a new
|
||||
# query for every kind and leaves a kind nobody added with no number;
|
||||
# grouping by task_kind counts every kind, including ones added later,
|
||||
# in two round-trips.
|
||||
non_task = _scoped(
|
||||
select(Note.note_type, func.count(Note.id))
|
||||
.where(Note.status.is_(None))
|
||||
.where(Note.note_type.in_(NON_TASK_FACETS))
|
||||
).group_by(Note.note_type)
|
||||
counts = {t: n for t, n in (await session.execute(non_task)).all()}
|
||||
|
||||
by_kind = _scoped(
|
||||
select(Note.task_kind, func.count(Note.id))
|
||||
.where(Note.status.isnot(None))
|
||||
).group_by(Note.task_kind)
|
||||
kind_counts = {k: n for k, n in (await session.execute(by_kind)).all()}
|
||||
|
||||
# Kinds are SUBSETS of 'task' and are deliberately left out of the total —
|
||||
# adding them would count every task twice.
|
||||
counts["task"] = sum(kind_counts.values())
|
||||
for kind, value in _FACETS.items():
|
||||
if value[0] and value[1] is not None:
|
||||
counts[kind] = kind_counts.get(kind, 0)
|
||||
|
||||
for t in NON_TASK_FACETS:
|
||||
counts.setdefault(t, 0)
|
||||
counts["total"] = counts["task"] + sum(counts[t] for t in NON_TASK_FACETS)
|
||||
return counts
|
||||
|
||||
|
||||
async def query_knowledge_ids(
|
||||
user_id: int,
|
||||
note_type: str | None,
|
||||
tags: list[str],
|
||||
sort: str,
|
||||
q: str | None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[int], int]:
|
||||
"""Return note IDs only — cheap query for the two-tier pagination feed."""
|
||||
if q:
|
||||
# Re-use semantic search, extract IDs in rank order
|
||||
items, total = await _semantic_knowledge_search(
|
||||
user_id, q, note_type=note_type, tags=tags,
|
||||
limit=limit, offset=offset,
|
||||
)
|
||||
return [item["id"] for item in items], total
|
||||
|
||||
# Browsing (see query_knowledge) — narrower scope.
|
||||
visible = browsable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
base = select(Note.id).where(visible)
|
||||
|
||||
base = _apply_type_filter(base, note_type)
|
||||
for tag in tags:
|
||||
base = base.where(Note.tags.contains([tag]))
|
||||
|
||||
count_stmt = select(func.count()).select_from(base.subquery())
|
||||
total: int = (await session.execute(count_stmt)).scalar_one()
|
||||
|
||||
if sort == "created":
|
||||
base = base.order_by(Note.created_at.desc())
|
||||
elif sort == "alpha":
|
||||
base = base.order_by(Note.title.asc())
|
||||
elif sort == "type":
|
||||
base = base.order_by(Note.note_type.asc(), Note.updated_at.desc())
|
||||
else:
|
||||
base = base.order_by(Note.updated_at.desc())
|
||||
|
||||
ids = list((await session.execute(base.limit(limit).offset(offset))).scalars().all())
|
||||
|
||||
return ids, total
|
||||
|
||||
|
||||
async def get_knowledge_by_ids(user_id: int, ids: list[int]) -> list[dict]:
|
||||
"""Fetch full items for the given IDs, preserving the requested order."""
|
||||
if not ids:
|
||||
return []
|
||||
# Fetching specific ids is explicit, so this takes the full read scope — the
|
||||
# ids came from either a browse or a search, and both must resolve.
|
||||
visible = readable_notes_clause(user_id)
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(Note)
|
||||
.where(visible)
|
||||
.where(Note.id.in_(ids))
|
||||
.where(Note.deleted_at.is_(None))
|
||||
)
|
||||
rows = list((await session.execute(stmt)).scalars().all())
|
||||
by_id = {n.id: n for n in rows}
|
||||
return [_note_to_item(by_id[i]) for i in ids if i in by_id]
|
||||
@@ -0,0 +1,922 @@
|
||||
"""Lessons point at rules.
|
||||
|
||||
A lesson is a non-binding record of one situation; a rule is the binding
|
||||
choice for a class of them. This service owns the link between the two — which
|
||||
rule a lesson is an instance of — and both directions of reading it.
|
||||
|
||||
THE STATES (models/lesson_rule_link.py says why each exists):
|
||||
`suggested` while evidence accumulates, `confirmed` or `rejected` once a
|
||||
judgment is made. Every write here is a JUDGMENT, so every write lands as
|
||||
confirmed or rejected; `suggested` rows come from the co-surfacing recorder,
|
||||
never from a caller naming a rule.
|
||||
|
||||
THE THIRD ANSWER. "Which rule is this an instance of?" has three
|
||||
answers, and only two of them are links: a rule named (confirmed), or NO RULE
|
||||
FITS — a `lesson_no_rule` row carrying the reason. A lesson with neither is
|
||||
UNJUDGED, and `list_unjudged` lists those. The three are kept apart because a
|
||||
lesson that stands alone is the raw material for a rule nobody has written yet
|
||||
(see Convergence below), and that is unreadable if "nobody looked" means the
|
||||
same thing.
|
||||
|
||||
THE SOFT LINK. When one hook request puts a lesson and a rule in
|
||||
front of the reader together, `co_surfaced` records the pair as evidence on a
|
||||
SUGGESTED link, and once that evidence crosses a bar the next co-arrival asks
|
||||
the reader to judge it. The evidence counts distinct SITUATIONS, not
|
||||
arrivals: two records that resemble each other arrive together on every
|
||||
similar prompt, and counting each time would measure their similarity over
|
||||
and over rather than any relation between them.
|
||||
|
||||
ACL (services/access.py). Linking changes what a lesson says about itself, so it needs
|
||||
WRITE on the lesson (`access.can_write_note`, share-aware). It names a rule, so
|
||||
it needs the rule to be one the caller may read — rules are owner-scoped, and
|
||||
`rulebooks._fetch_owned_rule` / `_owned_rules_clause` are that check's two
|
||||
forms. A read shows only the rules the READER owns: a lesson shared with
|
||||
someone must not hand them the titles of its owner's private rules.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import statistics
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, delete, exists, func, not_, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.lesson_rule_link import (
|
||||
CONFIRMED, REJECTED, SUGGESTED, LessonNoRule, LessonRuleLink,
|
||||
)
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.rulebook import Rule
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# What a caller says to judge one pair. Words rather than the stored states,
|
||||
# because "confirm" and "reject" are acts and the states are their results.
|
||||
VERDICTS = {"confirm": CONFIRMED, "reject": REJECTED}
|
||||
|
||||
# Recorded on a link that set-semantics removed, so a reader of the rejected
|
||||
# row can tell an explicit "not this rule" from a link dropped by a rewrite.
|
||||
_REMOVED_NOTE = "removed from the lesson's rules by an update"
|
||||
|
||||
# Recorded on a confirmed link that a "no rule fits" answer overturned, ahead
|
||||
# of that answer's reason.
|
||||
_NO_RULE_NOTE = "no rule fits: "
|
||||
|
||||
# What `rule_judgment` reads on a lesson payload — the three answers.
|
||||
LINKED, NO_RULE, UNJUDGED = "linked", "no_rule", "unjudged"
|
||||
|
||||
# THE SOFT-LINK EVIDENCE BAR — defaults, stated as defaults: nothing about an
|
||||
# install chose them.
|
||||
#
|
||||
# Three distinct situations before a pair is put to the reader. One is
|
||||
# coincidence and two is a pattern only by courtesy; three different prompts
|
||||
# or files that each brought the same lesson and rule up together is the
|
||||
# smallest count that says the pairing is not a single resemblance repeated.
|
||||
PROPOSE_SITUATIONS = 3
|
||||
# A proposal the reader passed over is asked again only after this long, so
|
||||
# one session's run of related prompts does not repeat the same question on
|
||||
# every turn.
|
||||
PROPOSE_COOLDOWN = timedelta(hours=6)
|
||||
# Fingerprints kept per link. The bar needs three; the rest is for the reader
|
||||
# judging later, and a list that grew with every situation would bloat a row
|
||||
# whose answer stopped changing long ago.
|
||||
_SITUATION_CAP = 50
|
||||
# A token shorter than this is grammar, not situation — "a", "is", "to".
|
||||
_MIN_TOKEN = 3
|
||||
|
||||
# How many rules a new lesson is offered to judge against. A handful, not the
|
||||
# fifty `what_might_apply` returns: this is read in the create response, at
|
||||
# the moment of writing, and a long tail there buries the one that fits.
|
||||
CANDIDATE_LIMIT = 5
|
||||
|
||||
|
||||
def _ids(values) -> list[int]:
|
||||
"""Positive ints, de-duplicated, in the order given."""
|
||||
out: list[int] = []
|
||||
for v in values or []:
|
||||
try:
|
||||
i = int(v)
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"rule id {v!r} is not an integer")
|
||||
if i > 0 and i not in out:
|
||||
out.append(i)
|
||||
return out
|
||||
|
||||
|
||||
async def _require_lesson_writable(user_id: int, lesson_id: int) -> None:
|
||||
from grimoire.services import access
|
||||
from grimoire.services import lessons as lessons_svc
|
||||
|
||||
note = await lessons_svc.get_lesson(user_id, lesson_id)
|
||||
if note is None:
|
||||
raise ValueError(f"lesson {lesson_id} not found")
|
||||
if not await access.can_write_note(user_id, lesson_id):
|
||||
raise PermissionError(f"lesson {lesson_id} is not yours to change")
|
||||
|
||||
|
||||
async def require_rules(user_id: int, rule_ids) -> list[int]:
|
||||
"""The ids, validated as rules the caller owns — all of them or none.
|
||||
|
||||
Checked BEFORE anything is written, by every caller, so a lesson create
|
||||
that names a rule it cannot see fails without leaving a half-linked
|
||||
lesson behind.
|
||||
"""
|
||||
from grimoire.services import rulebooks as rulebooks_svc
|
||||
|
||||
wanted = _ids(rule_ids)
|
||||
missing = [rid for rid in wanted if await rulebooks_svc.get_rule(rid, user_id) is None]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"rule(s) {missing} not found — a lesson can point only at a rule "
|
||||
"you can read. Nothing was linked."
|
||||
)
|
||||
return wanted
|
||||
|
||||
|
||||
def require_one_answer(linked, no_rule: str) -> None:
|
||||
"""Refuse "these rules fit" and "no rule fits" together, before any write
|
||||
— they contradict, and either order of applying them loses one."""
|
||||
if linked and (no_rule or "").strip():
|
||||
raise ValueError(
|
||||
"rule_ids and no_rule are the two answers to \"which rule is this "
|
||||
"an instance of?\" — give the one that holds. Nothing was written."
|
||||
)
|
||||
|
||||
|
||||
async def _upsert(session, lesson_id: int, rule_id: int, state: str, note: str) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
row = (await session.execute(
|
||||
select(LessonRuleLink).where(
|
||||
LessonRuleLink.lesson_id == lesson_id,
|
||||
LessonRuleLink.rule_id == rule_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if row is None:
|
||||
session.add(LessonRuleLink(
|
||||
lesson_id=lesson_id, rule_id=rule_id, state=state,
|
||||
note=note or None, judged_at=now,
|
||||
))
|
||||
return
|
||||
# Evidence is kept across a judgment: a confirmation reads best beside
|
||||
# what it rested on.
|
||||
row.state = state
|
||||
row.note = note or row.note
|
||||
row.judged_at = now
|
||||
|
||||
|
||||
async def _clear_no_rule(session, lesson_id: int) -> None:
|
||||
"""A confirmed rule overturns "no rule fits" — both cannot be the answer."""
|
||||
await session.execute(delete(LessonNoRule).where(LessonNoRule.lesson_id == lesson_id))
|
||||
|
||||
|
||||
async def set_lesson_rules(
|
||||
user_id: int, lesson_id: int, rule_ids, *, note: str = "",
|
||||
) -> None:
|
||||
"""Make the lesson's CONFIRMED rules exactly `rule_ids` (set-semantics).
|
||||
|
||||
A rule named here is confirmed, whatever state it was in: the writer of a
|
||||
lesson saying "this is an instance of rule N" is the judgment the
|
||||
suggested state waits for, so it needs no evidence bar.
|
||||
|
||||
A rule that WAS confirmed and is no longer named becomes `rejected`, not
|
||||
deleted. Dropping it is a judgment that the lesson is not an instance of
|
||||
that rule, and a deleted row would let the co-surfacing recorder propose
|
||||
the same pair again.
|
||||
"""
|
||||
await _require_lesson_writable(user_id, lesson_id)
|
||||
wanted = await require_rules(user_id, rule_ids)
|
||||
async with async_session() as session:
|
||||
current = (await session.execute(
|
||||
select(LessonRuleLink).where(
|
||||
LessonRuleLink.lesson_id == lesson_id,
|
||||
LessonRuleLink.state == CONFIRMED,
|
||||
)
|
||||
)).scalars().all()
|
||||
for row in current:
|
||||
if row.rule_id not in wanted:
|
||||
row.state = REJECTED
|
||||
row.note = _REMOVED_NOTE
|
||||
row.judged_at = datetime.now(timezone.utc)
|
||||
for rid in wanted:
|
||||
await _upsert(session, lesson_id, rid, CONFIRMED, note)
|
||||
if wanted:
|
||||
await _clear_no_rule(session, lesson_id)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def set_no_rule(user_id: int, lesson_id: int, why: str) -> dict:
|
||||
"""Record that no rule governs this lesson's situation, and why.
|
||||
|
||||
A confirmed link the lesson still carries becomes `rejected`, with the
|
||||
reason: saying none fits is saying the linked one does not either. A
|
||||
second answer replaces the first, so the reason on file is the latest.
|
||||
"""
|
||||
why = (why or "").strip()
|
||||
if not why:
|
||||
raise ValueError(
|
||||
"no_rule takes the reason no rule fits, in a line — it is what "
|
||||
"lets the answer be re-judged when a rule is later written for "
|
||||
"this situation."
|
||||
)
|
||||
await _require_lesson_writable(user_id, lesson_id)
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
confirmed = (await session.execute(
|
||||
select(LessonRuleLink).where(
|
||||
LessonRuleLink.lesson_id == lesson_id,
|
||||
LessonRuleLink.state == CONFIRMED,
|
||||
)
|
||||
)).scalars().all()
|
||||
for row in confirmed:
|
||||
row.state = REJECTED
|
||||
row.note = _NO_RULE_NOTE + why
|
||||
row.judged_at = now
|
||||
answer = await session.get(LessonNoRule, lesson_id)
|
||||
if answer is None:
|
||||
session.add(LessonNoRule(lesson_id=lesson_id, why=why, judged_at=now))
|
||||
else:
|
||||
answer.why = why
|
||||
answer.judged_at = now
|
||||
await session.commit()
|
||||
return {"why": why, "judged_at": now.isoformat()}
|
||||
|
||||
|
||||
async def judge_link(
|
||||
user_id: int, lesson_id: int, rule_id: int, verdict: str, note: str = "",
|
||||
) -> dict:
|
||||
"""Confirm or reject one (lesson, rule) pair, suggested or not."""
|
||||
state = VERDICTS.get((verdict or "").strip().lower())
|
||||
if state is None:
|
||||
raise ValueError(f"verdict must be one of {sorted(VERDICTS)}, got {verdict!r}")
|
||||
await _require_lesson_writable(user_id, lesson_id)
|
||||
[rid] = await require_rules(user_id, [rule_id])
|
||||
async with async_session() as session:
|
||||
await _upsert(session, lesson_id, rid, state, note)
|
||||
if state == CONFIRMED:
|
||||
await _clear_no_rule(session, lesson_id)
|
||||
await session.commit()
|
||||
row = (await session.execute(
|
||||
select(LessonRuleLink).where(
|
||||
LessonRuleLink.lesson_id == lesson_id,
|
||||
LessonRuleLink.rule_id == rid,
|
||||
)
|
||||
)).scalar_one()
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
async def rules_for_lessons(user_id: int, lesson_ids) -> dict[int, list[dict]]:
|
||||
"""{lesson_id: [{id, title, kind, state, note}]}, for rules the READER owns.
|
||||
|
||||
One query for the whole set — a lesson list would otherwise be N+1.
|
||||
Confirmed first, then suggested, then rejected: the order a reader cares
|
||||
about them in.
|
||||
"""
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
ids = [int(i) for i in lesson_ids or []]
|
||||
out: dict[int, list[dict]] = {i: [] for i in ids}
|
||||
if not ids:
|
||||
return out
|
||||
order = {s: n for n, s in enumerate((CONFIRMED, SUGGESTED, REJECTED))}
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LessonRuleLink, Rule)
|
||||
.join(Rule, Rule.id == LessonRuleLink.rule_id)
|
||||
.where(LessonRuleLink.lesson_id.in_(ids))
|
||||
.where(_owned_rules_clause(user_id))
|
||||
)).all()
|
||||
for link, rule in sorted(rows, key=lambda r: (order.get(r[0].state, 9), r[1].id)):
|
||||
item = {
|
||||
"id": rule.id, "title": rule.title, "kind": rule.kind,
|
||||
"state": link.state, "note": link.note or "",
|
||||
}
|
||||
if link.state == SUGGESTED:
|
||||
# What the suggestion rests on, so a reader can judge it from
|
||||
# here rather than having to go and count.
|
||||
item["evidence"] = evidence_summary(link.evidence)
|
||||
out[link.lesson_id].append(item)
|
||||
return out
|
||||
|
||||
|
||||
async def lessons_for_rule(user_id: int, rule_id: int) -> list[dict]:
|
||||
"""The lessons that point at one rule, readable by the caller (share-aware).
|
||||
|
||||
The reverse direction, and the one a rule's page needs: the concrete
|
||||
situations that have been judged instances of it.
|
||||
"""
|
||||
from grimoire.services.access import readable_notes_clause
|
||||
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LessonRuleLink, Note)
|
||||
.join(Note, Note.id == LessonRuleLink.lesson_id)
|
||||
.where(LessonRuleLink.rule_id == int(rule_id))
|
||||
.where(Note.deleted_at.is_(None))
|
||||
.where(readable_notes_clause(user_id))
|
||||
)).all()
|
||||
order = {s: n for n, s in enumerate((CONFIRMED, SUGGESTED, REJECTED))}
|
||||
return [
|
||||
{"id": note.id, "title": note.title, "state": link.state, "note": link.note or ""}
|
||||
for link, note in sorted(rows, key=lambda r: (order.get(r[0].state, 9), r[1].id))
|
||||
]
|
||||
|
||||
|
||||
async def judgments_for_lessons(lesson_ids) -> dict[int, dict]:
|
||||
"""{lesson_id: {"rule_judgment": linked|no_rule|unjudged, "no_rule"?}}.
|
||||
|
||||
Read from the link and answer tables directly, NOT from the reader's view
|
||||
of the rules: a lesson shared with someone who cannot see its owner's rule
|
||||
is still a judged lesson, and calling it unjudged would invite them to
|
||||
judge it again.
|
||||
"""
|
||||
ids = [int(i) for i in lesson_ids or []]
|
||||
if not ids:
|
||||
return {}
|
||||
async with async_session() as session:
|
||||
linked = set((await session.execute(
|
||||
select(LessonRuleLink.lesson_id).where(
|
||||
LessonRuleLink.lesson_id.in_(ids),
|
||||
LessonRuleLink.state == CONFIRMED,
|
||||
)
|
||||
)).scalars().all())
|
||||
answers = {
|
||||
a.lesson_id: a for a in (await session.execute(
|
||||
select(LessonNoRule).where(LessonNoRule.lesson_id.in_(ids))
|
||||
)).scalars().all()
|
||||
}
|
||||
out: dict[int, dict] = {}
|
||||
for i in ids:
|
||||
if i in linked:
|
||||
out[i] = {"rule_judgment": LINKED}
|
||||
elif i in answers:
|
||||
out[i] = {"rule_judgment": NO_RULE, "no_rule": answers[i].to_dict()}
|
||||
else:
|
||||
out[i] = {"rule_judgment": UNJUDGED}
|
||||
return out
|
||||
|
||||
|
||||
async def attach_lesson_rules(user_id: int, rows: list[dict], *, key: str = "id") -> None:
|
||||
"""Add `rules` and `rule_judgment` (plus `no_rule` when that is the answer)
|
||||
to each lesson payload row, in place — a fixed number of queries per page.
|
||||
|
||||
Fail-open: this decorates a lesson the caller already has,
|
||||
so a failed lookup leaves the keys off and logs, rather than refusing the
|
||||
lesson. An absent key reads as "not attached", never as "no rule".
|
||||
"""
|
||||
ids = [int(r[key]) for r in rows if isinstance(r.get(key), int)]
|
||||
try:
|
||||
found = await rules_for_lessons(user_id, ids)
|
||||
judged = await judgments_for_lessons(ids)
|
||||
except Exception:
|
||||
logger.warning("lesson→rule links could not be read", exc_info=True)
|
||||
return
|
||||
for r in rows:
|
||||
if isinstance(r.get(key), int):
|
||||
r["rules"] = found.get(r[key], [])
|
||||
r.update(judged.get(r[key], {}))
|
||||
|
||||
|
||||
def unjudged_clause():
|
||||
"""SQL: a lesson (a `notes` row) with no confirmed rule and no "no rule
|
||||
fits" answer. A rejected link alone leaves it unjudged — "not that rule"
|
||||
does not say whether another one fits."""
|
||||
from grimoire.models.note import Note
|
||||
|
||||
return and_(
|
||||
not_(exists().where(and_(
|
||||
LessonRuleLink.lesson_id == Note.id,
|
||||
LessonRuleLink.state == CONFIRMED,
|
||||
))),
|
||||
not_(exists().where(LessonNoRule.lesson_id == Note.id)),
|
||||
)
|
||||
|
||||
|
||||
async def list_unjudged(
|
||||
user_id: int, *, tag: str = "", project_id: int | None = None,
|
||||
limit: int = 50, offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""The lessons nobody has answered "which rule?" for, newest edit first.
|
||||
|
||||
The browse scope (`browsable_notes_clause`), the list it sits beside uses:
|
||||
a lesson shared directly with the caller is search-only and stays out of
|
||||
an ambient list. Rows come back in the knowledge list's item shape.
|
||||
"""
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.services.access import browsable_notes_clause
|
||||
from grimoire.services.knowledge import _note_to_item
|
||||
from grimoire.services.lessons import LESSON_NOTE_TYPE
|
||||
|
||||
base = (
|
||||
select(Note)
|
||||
.where(browsable_notes_clause(user_id))
|
||||
.where(Note.note_type == LESSON_NOTE_TYPE)
|
||||
.where(Note.deleted_at.is_(None))
|
||||
.where(unjudged_clause())
|
||||
)
|
||||
if tag:
|
||||
base = base.where(Note.tags.contains([tag]))
|
||||
if project_id is not None:
|
||||
base = base.where(Note.project_id == project_id)
|
||||
async with async_session() as session:
|
||||
total = (await session.execute(
|
||||
select(func.count()).select_from(base.subquery())
|
||||
)).scalar_one()
|
||||
rows = (await session.execute(
|
||||
base.order_by(Note.updated_at.desc()).limit(limit).offset(offset)
|
||||
)).scalars().all()
|
||||
return [_note_to_item(n) for n in rows], int(total)
|
||||
|
||||
|
||||
async def rule_candidates(
|
||||
user_id: int, what: str, when_to_apply: str, project_id: int | None = None,
|
||||
) -> list[dict] | None:
|
||||
"""The rules a lesson most resembles, for the writer to judge against.
|
||||
|
||||
Searched with the lesson's claim and its trigger — the same two halves a
|
||||
rule's own document leads with (`trigger_title`), so like is
|
||||
compared with like; the insight is the story and would dilute the vector.
|
||||
Scoped as any project read is: global rules plus the lesson's project's
|
||||
own. The bar is the explicit rule search's, since this is an explicit
|
||||
question rather than an injection.
|
||||
|
||||
None when the search could not run (no embedder, a failed query), so a
|
||||
caller can say "candidates unavailable" rather than "nothing resembles".
|
||||
"""
|
||||
from grimoire.services.embeddings import (
|
||||
DEFAULT_SIMILARITY_THRESHOLD, semantic_search_rules, trigger_title,
|
||||
)
|
||||
|
||||
# The rule document's own title join, so like is compared with like.
|
||||
query = trigger_title(what, when_to_apply)
|
||||
report: dict = {}
|
||||
found = await semantic_search_rules(
|
||||
user_id, query, limit=CANDIDATE_LIMIT,
|
||||
threshold=DEFAULT_SIMILARITY_THRESHOLD, report=report,
|
||||
project_id=project_id or None,
|
||||
)
|
||||
if not report.get("searched"):
|
||||
return None
|
||||
return [
|
||||
{
|
||||
"id": rule.id, "title": rule.title, "kind": rule.kind,
|
||||
"when_to_apply": rule.when_to_apply or "", "score": round(score, 3),
|
||||
}
|
||||
for score, rule in found
|
||||
]
|
||||
|
||||
|
||||
async def attach_rule_lessons(user_id: int, data: dict, rule_id: int) -> None:
|
||||
"""Add `lessons` to a rule payload, in place, when there are any.
|
||||
|
||||
Present-only, as a rule's `systems` and `relations` are, and
|
||||
fail-open for the reason `attach_lesson_rules` gives.
|
||||
"""
|
||||
try:
|
||||
lessons = await lessons_for_rule(user_id, rule_id)
|
||||
except Exception:
|
||||
logger.warning("rule→lesson links could not be read", exc_info=True)
|
||||
return
|
||||
if lessons:
|
||||
data["lessons"] = lessons
|
||||
|
||||
|
||||
|
||||
def situation_key(arm: str, text: str) -> str:
|
||||
"""A fingerprint of one situation, so a repeat counts once.
|
||||
|
||||
`arm` is part of the key — "p" for a prompt, "w" for a file being written,
|
||||
"s" for a session (the rule↔moment signal), "d" for a day (the misfire
|
||||
signal, whose door carries no session) —
|
||||
because each names a different kind of situation and must not collide.
|
||||
On the prompt arm the text is the prompt: lowercased, split into word
|
||||
tokens of _MIN_TOKEN or more, de-duplicated and sorted, so the same ask
|
||||
re-sent with different spacing, punctuation, case or word order is one
|
||||
situation. On the write arm the text is the PATH, not the code: every
|
||||
edit to one file is one situation, however much the code differs between
|
||||
them. The session and day arms key on the id or date as given — a date's
|
||||
"10" and "05" are shorter than _MIN_TOKEN, so tokenising it would make
|
||||
every day of a year one situation. Empty when there is nothing to key on.
|
||||
"""
|
||||
if arm in ("w", "s", "d"):
|
||||
basis = (text or "").strip()
|
||||
else:
|
||||
tokens = sorted({t for t in re.findall(r"[a-z0-9]+", (text or "").lower())
|
||||
if len(t) >= _MIN_TOKEN})
|
||||
basis = " ".join(tokens)
|
||||
if not basis:
|
||||
return ""
|
||||
return f"{arm}:" + hashlib.sha1(basis.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def evidence_summary(evidence) -> dict:
|
||||
"""The counts a reader judges a suggested link by."""
|
||||
ev = evidence if isinstance(evidence, dict) else {}
|
||||
return {
|
||||
"situations": len(ev.get("situations") or []),
|
||||
"projects": len(ev.get("projects") or []),
|
||||
"co_surfaced": int(ev.get("co_surfaced") or 0),
|
||||
}
|
||||
|
||||
|
||||
def add_evidence(evidence, key: str, project_id: int | None, now: datetime) -> dict:
|
||||
"""A NEW evidence dict with this co-arrival counted. Pure, so the counting
|
||||
rules are testable without a database; a new dict, so SQLAlchemy sees the
|
||||
JSONB column change."""
|
||||
ev = dict(evidence) if isinstance(evidence, dict) else {}
|
||||
ev["co_surfaced"] = int(ev.get("co_surfaced") or 0) + 1
|
||||
situations = list(ev.get("situations") or [])
|
||||
if key and key not in situations and len(situations) < _SITUATION_CAP:
|
||||
situations.append(key)
|
||||
ev["situations"] = situations
|
||||
projects = list(ev.get("projects") or [])
|
||||
if project_id and int(project_id) not in projects:
|
||||
projects.append(int(project_id))
|
||||
ev["projects"] = projects
|
||||
ev.setdefault("first_at", now.isoformat())
|
||||
ev["last_at"] = now.isoformat()
|
||||
return ev
|
||||
|
||||
|
||||
def proposal_due(evidence, now: datetime) -> bool:
|
||||
"""At the bar, and not asked within the cooldown."""
|
||||
ev = evidence if isinstance(evidence, dict) else {}
|
||||
if len(ev.get("situations") or []) < PROPOSE_SITUATIONS:
|
||||
return False
|
||||
last = ev.get("proposed_at")
|
||||
if not last:
|
||||
return True
|
||||
try:
|
||||
return now - datetime.fromisoformat(last) >= PROPOSE_COOLDOWN
|
||||
except (TypeError, ValueError):
|
||||
return True
|
||||
|
||||
|
||||
def _proposal_line(lesson_id: int, lesson_title: str, rule_id: int, rule_title: str,
|
||||
kind: str, evidence: dict) -> str:
|
||||
counts = evidence_summary(evidence)
|
||||
where = (f" across {counts['projects']} projects" if counts["projects"] > 1 else "")
|
||||
return (
|
||||
f"> Lesson #{lesson_id} \"{lesson_title}\" and {kind} #{rule_id} "
|
||||
f"\"{rule_title}\" keep arriving together — {counts['situations']} "
|
||||
f"distinct situations{where}. If the lesson is an instance of the "
|
||||
f"{kind}, `judge_lesson_link({lesson_id}, {rule_id}, \"confirm\", "
|
||||
f"note=\"why\")` links them and the {kind} becomes reachable through "
|
||||
f"the lesson's situation; if it is not, `\"reject\"` with the why "
|
||||
f"stops the pair being asked about again."
|
||||
)
|
||||
|
||||
|
||||
async def co_surfaced(
|
||||
user_id: int, lesson_ids, rule_ids, *, arm: str, situation: str,
|
||||
project_id: int | None = None,
|
||||
) -> str:
|
||||
"""Record that these lessons and rules arrived together in one request,
|
||||
and return a proposal line when a pair has earned one ("" otherwise).
|
||||
|
||||
Called by the hook doors that put both kinds in front of the reader in
|
||||
the SAME response — `/retrieve` on a prompt, `/prior-art` on a write — so
|
||||
the pairing is exact: nothing is joined across requests, and no session
|
||||
identity is needed.
|
||||
|
||||
Only pairs the reader could confirm are recorded: a lesson they may write
|
||||
and a rule they own, the two checks `judge_link` makes. A pair already
|
||||
judged (confirmed or rejected) gathers nothing more — a confirmation needs
|
||||
no evidence, and a rejection is what stops the asking.
|
||||
|
||||
One proposal per request at most, so a menu never becomes a queue of
|
||||
questions. Fails open: this rides a hook, and a recall aid must never
|
||||
break the prompt or the write it decorates.
|
||||
"""
|
||||
try:
|
||||
return await _co_surfaced(user_id, lesson_ids, rule_ids, arm, situation, project_id)
|
||||
except Exception:
|
||||
logger.warning("lesson/rule co-surfacing could not be recorded", exc_info=True)
|
||||
return ""
|
||||
|
||||
|
||||
async def _co_surfaced(user_id, lesson_ids, rule_ids, arm, situation, project_id) -> str:
|
||||
from grimoire.services.access import can_write_note
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
lessons = _ids(lesson_ids)
|
||||
rules = _ids(rule_ids)
|
||||
key = situation_key(arm, situation)
|
||||
if not lessons or not rules or not key:
|
||||
return ""
|
||||
lessons = [lid for lid in lessons if await can_write_note(user_id, lid)]
|
||||
if not lessons:
|
||||
return ""
|
||||
now = datetime.now(timezone.utc)
|
||||
proposal = None
|
||||
async with async_session() as session:
|
||||
owned = {
|
||||
r.id: r for r in (await session.execute(
|
||||
select(Rule).where(Rule.id.in_(rules)).where(_owned_rules_clause(user_id))
|
||||
)).scalars().all()
|
||||
}
|
||||
if not owned:
|
||||
return ""
|
||||
existing = {
|
||||
(row.lesson_id, row.rule_id): row for row in (await session.execute(
|
||||
select(LessonRuleLink).where(
|
||||
LessonRuleLink.lesson_id.in_(lessons),
|
||||
LessonRuleLink.rule_id.in_(list(owned)),
|
||||
)
|
||||
)).scalars().all()
|
||||
}
|
||||
for lid in lessons:
|
||||
for rid in owned:
|
||||
row = existing.get((lid, rid))
|
||||
if row is not None and row.state != SUGGESTED:
|
||||
continue
|
||||
if row is None:
|
||||
row = LessonRuleLink(lesson_id=lid, rule_id=rid, state=SUGGESTED)
|
||||
session.add(row)
|
||||
ev = add_evidence(row.evidence, key, project_id, now)
|
||||
if proposal is None and proposal_due(ev, now):
|
||||
ev["proposed_at"] = now.isoformat()
|
||||
ev["proposed_count"] = int(ev.get("proposed_count") or 0) + 1
|
||||
proposal = (lid, rid, ev)
|
||||
row.evidence = ev
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Two hook requests created the same pair at once; the other
|
||||
# one's row stands, and this co-arrival is simply not counted.
|
||||
await session.rollback()
|
||||
return ""
|
||||
if proposal is None:
|
||||
return ""
|
||||
lid, rid, ev = proposal
|
||||
lesson_title = (await session.execute(
|
||||
select(Note.title).where(Note.id == lid)
|
||||
)).scalar_one_or_none() or ""
|
||||
rule = owned[rid]
|
||||
return _proposal_line(lid, lesson_title, rid, rule.title, rule.kind or "rule", ev)
|
||||
|
||||
|
||||
# ── Reaching a rule through its lessons ──────────────────────────────────────
|
||||
|
||||
|
||||
async def confirmed_lessons(user_id: int) -> set[int]:
|
||||
"""The lessons, readable by the caller, that carry a CONFIRMED link to a
|
||||
rule the caller owns — the only lessons that can bring a rule along.
|
||||
|
||||
CONFIRMED ONLY, and the reason is the soft link's: a suggested link that
|
||||
carried its rule would put the two side by side on every prompt that
|
||||
found the lesson, which is exactly the co-arrival the soft link counts
|
||||
as evidence — the suggestion would manufacture its own proof.
|
||||
|
||||
Asked first and cheaply, so an arm on an install with no confirmed links
|
||||
— every install, until someone judges one — never pays for a search.
|
||||
"""
|
||||
from grimoire.services.access import readable_notes_clause
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LessonRuleLink.lesson_id)
|
||||
.join(Rule, Rule.id == LessonRuleLink.rule_id)
|
||||
.join(Note, Note.id == LessonRuleLink.lesson_id)
|
||||
.where(LessonRuleLink.state == CONFIRMED)
|
||||
.where(Rule.deleted_at.is_(None))
|
||||
.where(_owned_rules_clause(user_id))
|
||||
.where(Note.deleted_at.is_(None))
|
||||
.where(readable_notes_clause(user_id))
|
||||
.distinct()
|
||||
)).scalars().all()
|
||||
return {int(i) for i in rows}
|
||||
|
||||
|
||||
async def confirmed_rules_in_scope(
|
||||
user_id: int, lesson_ids, project_id: int | None,
|
||||
) -> dict[int, list]:
|
||||
"""{lesson_id: [Rule]} — each lesson's CONFIRMED rules that may surface here.
|
||||
|
||||
"May surface here" is the rule arms' own scope: a global rule, or a rule
|
||||
of THIS project. A lesson is project-independent and surfaces everywhere,
|
||||
so without this a lesson linked to project A's rule would carry that rule
|
||||
into project B's sessions — a leak the direct arms' scope prevents,
|
||||
reopened through a side door.
|
||||
"""
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
ids = [int(i) for i in lesson_ids or []]
|
||||
if not ids:
|
||||
return {}
|
||||
home = (
|
||||
or_(Rule.topic_id.isnot(None), Rule.project_id == project_id)
|
||||
if project_id else Rule.topic_id.isnot(None)
|
||||
)
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LessonRuleLink.lesson_id, Rule)
|
||||
.join(Rule, Rule.id == LessonRuleLink.rule_id)
|
||||
.where(LessonRuleLink.lesson_id.in_(ids))
|
||||
.where(LessonRuleLink.state == CONFIRMED)
|
||||
.where(Rule.deleted_at.is_(None))
|
||||
.where(_owned_rules_clause(user_id))
|
||||
.where(home)
|
||||
.order_by(Rule.id)
|
||||
)).all()
|
||||
out: dict[int, list] = {}
|
||||
for lesson_id, rule in rows:
|
||||
out.setdefault(int(lesson_id), []).append(rule)
|
||||
return out
|
||||
|
||||
|
||||
# ── Convergence: lessons with no rule that keep landing in one place ────────
|
||||
#
|
||||
# A lesson answered "no rule fits" is a situation nothing binds. One is a
|
||||
# lesson. Several that resemble each other are what a missing rule looks like
|
||||
# from the outside — the same moment met again and again, each time written
|
||||
# down as advice. This is noticed at the WRITE, when the newest of them is
|
||||
# answered, and never by a sweep or a timer: the reader is in the
|
||||
# situation then, and a nudge arriving anywhere else is one nobody acts on.
|
||||
#
|
||||
# WHAT "RESEMBLE" MEANS. Not a fixed similarity. Lessons are written
|
||||
# in one register — a sentence of advice and the moment it applies — so an
|
||||
# embedder scores any two of them well above two unrelated texts, and a
|
||||
# constant bar sits inside that band: every no-rule lesson then "resembles"
|
||||
# most of the others, and the group grows with the pool rather than with any
|
||||
# situation recurring. The bar is instead relative to each lesson's own
|
||||
# background: how far a neighbour stands above the typical score that lesson
|
||||
# gets against every lesson in its register, measured in that register's own
|
||||
# spread (median and MAD, so the near neighbours being judged do not move the
|
||||
# yardstick). An install with a different embedder, or lessons in a different
|
||||
# style, gets a different band and the same bar.
|
||||
#
|
||||
# And resemblance must be MUTUAL, among every member, not just to the lesson
|
||||
# being answered. A lesson broad enough to stand near many others is a hub,
|
||||
# and hub-and-spoke lets three unrelated lessons make a "group" through one
|
||||
# general one. A clique is the shape of one situation met again.
|
||||
#
|
||||
# DEFAULTS, stated as defaults. Three lessons — the new one
|
||||
# and two it resembles — is the smallest group that is a pattern rather than
|
||||
# a pair. One spread above the background is a modest bar for one direction
|
||||
# alone; the strictness comes from requiring it of every pair in both
|
||||
# directions, which unrelated lessons rarely all clear. Raise it if groups
|
||||
# name lessons a reader would not call one situation; lower it if lessons a
|
||||
# reader would group never meet.
|
||||
CONVERGENCE_LESSONS = 3
|
||||
CONVERGENCE_STANDOUT = 1.0
|
||||
# Below this many other lessons a median and its spread describe too little
|
||||
# to say what stands out, so nothing is named.
|
||||
CONVERGENCE_MIN_BACKGROUND = 8
|
||||
# How many of a lesson's register the background is read from. A register
|
||||
# larger than this is measured on its nearest part, which reads the
|
||||
# background HIGH — the bar errs strict, toward naming nothing, never toward
|
||||
# a group that is only the pool's size.
|
||||
_CONVERGENCE_BACKGROUND = 200
|
||||
# Neighbours checked for mutual resemblance, nearest first. Each costs one
|
||||
# more search at the write, and a group large enough to need more is named
|
||||
# just as well by its nearest members.
|
||||
_CONVERGENCE_CANDIDATES = 8
|
||||
# The consistency constant that makes a MAD estimate a standard deviation's
|
||||
# scale on normal data, so CONVERGENCE_STANDOUT reads as "spreads".
|
||||
_MAD_SCALE = 1.4826
|
||||
|
||||
|
||||
def convergence_group(members: list[dict]) -> dict | None:
|
||||
"""Decide from a candidate group — the new lesson FIRST, then those it
|
||||
resembles — whether it names a missing rule. Pure, so the bar is testable.
|
||||
|
||||
Each member is {id, title, sources, project_id}. The bar counts DISTINCT
|
||||
LESSONS, never incidents: a single broad lesson drawn from many incidents
|
||||
is still one judgment about one situation, and incidents cannot stand in
|
||||
for lessons. And when every member says what taught it and all of them
|
||||
point at the same single incident, that is one event written up several
|
||||
times, not a situation recurring — the group stays quiet.
|
||||
"""
|
||||
seen: set[int] = set()
|
||||
group = []
|
||||
for m in members:
|
||||
if m["id"] not in seen:
|
||||
seen.add(m["id"])
|
||||
group.append(m)
|
||||
if len(group) < CONVERGENCE_LESSONS:
|
||||
return None
|
||||
incidents = sorted({s for m in group for s in (m.get("sources") or [])})
|
||||
if all(m.get("sources") for m in group) and len(incidents) < 2:
|
||||
return None
|
||||
projects = sorted({m["project_id"] for m in group if m.get("project_id")})
|
||||
named = ", ".join(f"#{m['id']} “{m['title']}”" for m in group)
|
||||
return {
|
||||
"lessons": [{"id": m["id"], "title": m["title"]} for m in group],
|
||||
"incidents": incidents,
|
||||
"projects": projects,
|
||||
"hint": (
|
||||
f"{len(group)} lessons answered \"no rule fits\" and keep landing "
|
||||
f"in one situation: {named}. A situation met this often may want a "
|
||||
"rule — the binding choice they each circle. Draft it with "
|
||||
"create_rule (it goes to the operator, as every rule does), then "
|
||||
"point each lesson at it with update_lesson(lesson_id, "
|
||||
"rule_ids=[<new rule id>]). If no single choice is right every "
|
||||
"time, the lessons are the right record and nothing more is needed."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def _no_rule_ids(lesson_ids) -> set[int]:
|
||||
ids = [int(i) for i in lesson_ids or []]
|
||||
if not ids:
|
||||
return set()
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(LessonNoRule.lesson_id).where(LessonNoRule.lesson_id.in_(ids))
|
||||
)).scalars().all()
|
||||
return {int(i) for i in rows}
|
||||
|
||||
|
||||
async def convergence_for(user_id: int, lesson_id: int) -> dict | None:
|
||||
"""The convergence a newly answered no-rule lesson completes, or None.
|
||||
|
||||
Fail-open: this decorates a write that already succeeded, so a failed
|
||||
search leaves the response as it was.
|
||||
"""
|
||||
try:
|
||||
return await _convergence_for(user_id, lesson_id)
|
||||
except Exception:
|
||||
logger.warning("lesson convergence could not be checked", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def standout(scores: dict[int, float]) -> dict[int, float]:
|
||||
"""How far each lesson stands above this one's background, in spreads.
|
||||
|
||||
`scores` is one lesson's similarity to every other lesson in its register,
|
||||
itself left out. Pure, so the bar is testable. Empty when there is too
|
||||
little background to measure, or none of it varies.
|
||||
"""
|
||||
if len(scores) < CONVERGENCE_MIN_BACKGROUND:
|
||||
return {}
|
||||
middle = statistics.median(scores.values())
|
||||
spread = statistics.median(abs(v - middle) for v in scores.values()) * _MAD_SCALE
|
||||
if spread <= 0:
|
||||
return {}
|
||||
return {i: (v - middle) / spread for i, v in scores.items()}
|
||||
|
||||
|
||||
def converging(lesson_id: int, standouts: dict[int, dict[int, float]],
|
||||
candidates: list[int]) -> list[int]:
|
||||
"""The lesson, then each candidate that stands out to EVERY member so far
|
||||
and every member to it. Candidates are taken nearest first, so the clique
|
||||
grows around the closest situation rather than the first id. Pure."""
|
||||
def near(a: int, b: int) -> bool:
|
||||
return (standouts.get(a, {}).get(b, float("-inf")) >= CONVERGENCE_STANDOUT
|
||||
and standouts.get(b, {}).get(a, float("-inf")) >= CONVERGENCE_STANDOUT)
|
||||
|
||||
group = [lesson_id]
|
||||
for c in candidates:
|
||||
if c not in group and all(near(c, m) for m in group):
|
||||
group.append(c)
|
||||
return group
|
||||
|
||||
|
||||
async def _convergence_for(user_id: int, lesson_id: int) -> dict | None:
|
||||
from grimoire.services import lessons as lessons_svc
|
||||
from grimoire.services.embeddings import semantic_search_notes, trigger_title
|
||||
|
||||
lesson = await lessons_svc.get_lesson(user_id, lesson_id)
|
||||
if lesson is None:
|
||||
return None
|
||||
|
||||
async def background_of(note) -> list[tuple[float, Note]]:
|
||||
"""`note`'s similarity to every other lesson it can reach — the
|
||||
background it is measured against. No threshold: the bar is relative,
|
||||
and needs the scores a fixed bar would have turned away. No
|
||||
supersession penalty either: it would shift some scores and not
|
||||
others, and the background is a measure of resemblance alone."""
|
||||
data = note.data if isinstance(note.data, dict) else {}
|
||||
query = trigger_title(data.get("what") or note.title, lessons_svc.lesson_trigger(note))
|
||||
return await semantic_search_notes(
|
||||
user_id, query, exclude_ids={int(note.id)}, limit=_CONVERGENCE_BACKGROUND,
|
||||
threshold=-1.0, note_type=(lessons_svc.LESSON_NOTE_TYPE,),
|
||||
include_global_kinds=True, scope="browse", demote_superseded=False,
|
||||
)
|
||||
|
||||
lid = int(lesson.id)
|
||||
found = await background_of(lesson)
|
||||
notes = {int(n.id): n for _s, n in found}
|
||||
standouts = {lid: standout({int(n.id): s for s, n in found})}
|
||||
standing = [i for i, z in standouts[lid].items() if z >= CONVERGENCE_STANDOUT]
|
||||
answered = await _no_rule_ids(standing)
|
||||
candidates = sorted(answered, key=lambda i: (-standouts[lid][i], i))[:_CONVERGENCE_CANDIDATES]
|
||||
# Too few stand out from here for any clique to reach the bar — skip the
|
||||
# searches the mutual check would cost.
|
||||
if len(candidates) < CONVERGENCE_LESSONS - 1:
|
||||
return None
|
||||
for c in candidates:
|
||||
standouts[c] = standout({int(n.id): s for s, n in await background_of(notes[c])})
|
||||
|
||||
def member(note) -> dict:
|
||||
return {
|
||||
"id": int(note.id), "title": note.title,
|
||||
"sources": lessons_svc.lesson_sources(note), "project_id": note.project_id,
|
||||
}
|
||||
|
||||
group = converging(lid, standouts, candidates)
|
||||
return convergence_group([member(lesson)] + [member(notes[i]) for i in group[1:]])
|
||||
@@ -0,0 +1,651 @@
|
||||
"""Lesson service — a transferable insight, retrievable by situation.
|
||||
|
||||
A *lesson* is a Note with ``note_type='lesson'``: a better way to think about a
|
||||
problem, or a solution that transfers, recorded so a later session meets it at
|
||||
the moment it applies — and **without binding the reader**.
|
||||
|
||||
WHY THE KIND EXISTS
|
||||
|
||||
Without it, rules are the only surface that is global AND situation-keyed, so
|
||||
an agent holding a transferable insight has one door, and that door binds —
|
||||
and sessions propose rules for things that should not be rules.
|
||||
|
||||
The gap is a document-shape fact, not a threshold:
|
||||
|
||||
- a note is embedded as ``title\\nbody`` and is findable by **what it is
|
||||
about**;
|
||||
- a rule is embedded as ``{title} — {trigger}`` with ``When to apply:``
|
||||
repeated at the head of the body, so the trigger appears twice in a short
|
||||
document and dominates the vector — findable by **when it applies**.
|
||||
|
||||
No tuning reaches across that: the field the query would match on is simply not
|
||||
in a note's document. So a lesson carries a trigger and is embedded like a rule,
|
||||
while staying a note in every other respect.
|
||||
|
||||
WHERE THE TRIGGER LIVES
|
||||
|
||||
In ``notes.data`` under ``when_to_apply``, written through a named parameter and
|
||||
mirrored into the head of the body — the shape snippets use for
|
||||
``when_to_use``. Not a column on ``notes``. It is NOT in the stored title: the
|
||||
title is the lesson's subject, and the trigger joins it only in the embedded
|
||||
document (``embeddings.document_title``).
|
||||
|
||||
A named structured field rather than a column: snippets show such a field gets
|
||||
filled reliably without a guard, and a column would decide enforceability for
|
||||
every note kind at once.
|
||||
|
||||
The trigger in the document is what makes the vector sharp. It reaches it
|
||||
twice — in the embedded title, joined at embed time, and in the body's first
|
||||
line.
|
||||
|
||||
WHAT A LESSON INHERITS, AND THE CELLS LEFT EMPTY ON PURPOSE
|
||||
|
||||
A new kind inherits the note machinery wholesale. Which parts it should NOT get
|
||||
is stated here, so that an empty cell is a decision rather than an oversight.
|
||||
|
||||
Inherited, all deliberately:
|
||||
|
||||
- **versions** — a lesson is reworded as understanding improves, and what it
|
||||
used to say is worth as much as any note's history.
|
||||
- **supersession** — the event this most needs. A lesson replaced by a better
|
||||
lesson is precisely what ``note_supersessions`` models, and the demotion
|
||||
penalty already exists.
|
||||
- **trash**, **the share ACL**, **tags**, **project and System tagging**,
|
||||
**chunked embeddings**, **the near-duplicate gate**.
|
||||
|
||||
NOT inherited, and each for a stated reason:
|
||||
|
||||
- **status / task_kind / milestone_id** — a lesson is not work. ``is_task`` is
|
||||
``status is not None``, so a lesson that acquired a status would become a
|
||||
task and appear in open-work listings. This is the one cell where filling it
|
||||
in by accident silently changes what the record IS.
|
||||
- **recurrence** — task-only, and a lesson does not recur.
|
||||
- **verify_with / expires_when** — available, because they are generic note
|
||||
fields, but not part of a lesson's contract and not asked for on create.
|
||||
Those mark a record that asserts a FACT about someone else's software and
|
||||
can go false unwatched. A lesson is closer
|
||||
to a norm — "a better way to think about this" has no truth value that rots
|
||||
on its own. A lesson that does assert such a fact can still carry them.
|
||||
|
||||
WHY A KIND NEEDS NO MIGRATION
|
||||
|
||||
``note_type`` carries **no CHECK constraint** — only ``task_kind`` does
|
||||
(``notes_task_kind_check``). It is plain ``Text`` with a server default, so a
|
||||
value the database refuses on an un-migrated instance cannot arise for this
|
||||
column.
|
||||
|
||||
The real vocabulary is ``services.knowledge._FACETS``, which is where a kind
|
||||
becomes reachable on the browse surface and validated at the door. That is one
|
||||
table feeding both dialects of the type filter, so adding a kind there is a
|
||||
single edit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
|
||||
LESSON_NOTE_TYPE = "lesson"
|
||||
|
||||
# The key in `notes.data`. Named for the field it mirrors on `rules`, because it
|
||||
# answers the same question and a reader who knows one should not have to learn
|
||||
# a second word for it.
|
||||
TRIGGER_KEY = "when_to_apply"
|
||||
|
||||
# What taught this lesson: the ids of the issues, tasks or notes it was drawn
|
||||
# from. A LIST, and that is the whole decision — see `normalize_sources`.
|
||||
SOURCES_KEY = "taught_by"
|
||||
|
||||
# The body's trigger line, and the pattern that reads it back. The body is the
|
||||
# readable form and the thing that gets embedded; `data` is the queryable
|
||||
# mirror. Reads prefer the mirror and fall back to this, which is the discipline
|
||||
# `snippet_fields` follows and the reason a row written before the mirror
|
||||
# existed is still readable.
|
||||
_BODY_TRIGGER_RE = re.compile(r"^\*\*When to apply:\*\*\s*(.+?)\s*$", re.M)
|
||||
|
||||
# The readable mirror of `data[SOURCES_KEY]`, and the pattern that reads it
|
||||
# back — the shape snippets use for `**Merged from:** #ids`, for the same
|
||||
# reason: the body is what a human sees and what survives a row with no `data`.
|
||||
_BODY_SOURCES_RE = re.compile(r"^\*\*Learned from:\*\*\s*(.+?)\s*$", re.M)
|
||||
_ID_RE = re.compile(r"#(\d+)")
|
||||
|
||||
|
||||
def lesson_trigger(note) -> str:
|
||||
"""When this lesson applies, or "" — the mirror first, then the body.
|
||||
|
||||
Prefers `data` for the same reason every snippet read does: it is indexed,
|
||||
and parsing a body to answer a question the database can answer is how a
|
||||
hot path ends up regexing markdown. The fallback is not dead code — it is
|
||||
what makes a lesson readable if the mirror is ever absent, and an absent
|
||||
mirror must degrade to the right answer rather than to silence.
|
||||
"""
|
||||
data = getattr(note, "data", None) or {}
|
||||
from_mirror = (data.get(TRIGGER_KEY) or "").strip() if isinstance(data, dict) else ""
|
||||
if from_mirror:
|
||||
return from_mirror
|
||||
match = _BODY_TRIGGER_RE.search(getattr(note, "body", None) or "")
|
||||
return match.group(1).strip() if match else ""
|
||||
|
||||
|
||||
def normalize_sources(entries: list | None) -> list[int]:
|
||||
"""The ids that taught this lesson — ints, de-duplicated, in the order given.
|
||||
|
||||
THE CARDINALITY DECISION, and why it is a list.
|
||||
|
||||
`arose_from_id` already exists and holds ONE id, which is the obvious first
|
||||
answer and the wrong one. A good lesson often generalises several
|
||||
incidents — a badge collision, an un-backfilled column, a duplicated const
|
||||
— into one claim about a failure class. That is the shape a good lesson
|
||||
HAS, not an edge case. A single
|
||||
id would keep the first and silently drop the rest, and a record that drops
|
||||
two of its three sources is worse than one that names none, because it reads
|
||||
as complete.
|
||||
|
||||
It lives in `notes.data` rather than a join table for the reason the
|
||||
trigger lives there: a join table would settle, for every note kind at
|
||||
once, whether provenance is multi-valued. `data` is JSONB with a GIN
|
||||
index, so the list is
|
||||
queryable today and a table can be migrated to later if the need is shown.
|
||||
|
||||
Order is history, not sorting: the incidents stay in the sequence the writer
|
||||
named them, which is the order they were learned in.
|
||||
"""
|
||||
out: list[int] = []
|
||||
seen: set[int] = set()
|
||||
for raw in entries or []:
|
||||
ident = raw.get("id") if isinstance(raw, dict) else raw
|
||||
try:
|
||||
i = int(ident)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if i > 0 and i not in seen:
|
||||
seen.add(i)
|
||||
out.append(i)
|
||||
return out
|
||||
|
||||
|
||||
def lesson_sources(note) -> list[int]:
|
||||
"""What taught this lesson — the mirror first, then the body, then
|
||||
`arose_from_id`.
|
||||
|
||||
Three fallbacks rather than two, because the third is what keeps this
|
||||
honest on a record written before the kind existed: a note carrying only
|
||||
`arose_from_id` has exactly one source and this returns it, so a caller
|
||||
never has to ask which field to read.
|
||||
"""
|
||||
data = getattr(note, "data", None) or {}
|
||||
if isinstance(data, dict):
|
||||
from_mirror = normalize_sources(data.get(SOURCES_KEY))
|
||||
if from_mirror:
|
||||
return from_mirror
|
||||
match = _BODY_SOURCES_RE.search(getattr(note, "body", None) or "")
|
||||
if match:
|
||||
found = normalize_sources(_ID_RE.findall(match.group(1)))
|
||||
if found:
|
||||
return found
|
||||
single = getattr(note, "arose_from_id", None)
|
||||
return normalize_sources([single]) if single else []
|
||||
|
||||
|
||||
def sole_source(sources: list[int] | None) -> int | None:
|
||||
"""`arose_from_id` for this lesson: the id when there is exactly ONE.
|
||||
|
||||
Left NULL for a lesson drawn from several, deliberately. Every existing
|
||||
surface that renders provenance reads `arose_from_id` and renders it as
|
||||
THE origin; handing it one of three would make those surfaces state
|
||||
something false. Showing nothing there is accurate — there is no single
|
||||
origin — and `data[SOURCES_KEY]` carries all of them for the surfaces that
|
||||
know to ask.
|
||||
"""
|
||||
ids = normalize_sources(sources)
|
||||
return ids[0] if len(ids) == 1 else None
|
||||
|
||||
|
||||
def compose_body(
|
||||
insight: str, when_to_apply: str = "", learned_from: list[int] | None = None,
|
||||
) -> str:
|
||||
"""The lesson body — the trigger line first, the insight after.
|
||||
|
||||
The trigger's home in the text of the document, and half of what makes a
|
||||
lesson findable: `chunk_document` joins `{embedded title}\\n{body}`, and
|
||||
the embedded title is `what — when it applies` (`embeddings.document_title`),
|
||||
so the document states WHEN IT APPLIES in the title and again in the first
|
||||
line of the body. That twice-in-a-short-document shape is what measures
|
||||
sharp in the corpus.
|
||||
|
||||
`**When to apply:**` rather than plain text: the body is the READABLE
|
||||
form, `data` is the queryable mirror, and `_BODY_TRIGGER_RE` reads this
|
||||
line back when the mirror is missing. Its markdown must therefore match
|
||||
what that pattern expects, which is why neither is written by hand
|
||||
anywhere else.
|
||||
|
||||
The insight goes in the body rather than being held out of the document.
|
||||
`rule_document` excludes a rule's `why` because long narrative in ONE
|
||||
vector lands on the centroid of "development" — but a body over the
|
||||
budget is split into several chunks, EACH prefixed with the title, so a
|
||||
lesson's story does not average itself into its trigger: it occupies its
|
||||
own vectors, and every
|
||||
one of them still carries the trigger in its prefix. Holding it out would
|
||||
cost the reader the only part that explains the insight and would buy a
|
||||
sharpness the chunker already provides.
|
||||
"""
|
||||
lines = []
|
||||
trigger = (when_to_apply or "").strip()
|
||||
if trigger:
|
||||
lines.append(f"**When to apply:** {trigger}")
|
||||
insight = (insight or "").strip()
|
||||
if insight:
|
||||
lines.append(insight)
|
||||
sources = normalize_sources(learned_from)
|
||||
if sources:
|
||||
# LAST, not beside the trigger. The first line has to be what this
|
||||
# lesson is FOR; a provenance line above the insight would push the
|
||||
# thing the reader came for below a list of ids, and would put
|
||||
# numbers where the trigger's second appearance does its work.
|
||||
lines.append("**Learned from:** " + ", ".join(f"#{i}" for i in sources))
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
# A lesson's NAME is its claim, and the claim is one line. `what` is the title
|
||||
# every listing, menu and search row prints, so a story written there instead
|
||||
# of in `insight` turns each of those lines into a document, with the claim it
|
||||
# exists to transfer somewhere in the middle. It also ranks badly: the embedded title
|
||||
# is `what — when_to_apply`, so a narrative crowds out the trigger.
|
||||
#
|
||||
# 240 is a long sentence. The bound is on what a NAME can be, not on what a
|
||||
# lesson can say — `insight` is the body, unbounded, chunked like any record,
|
||||
# and costs the ranking nothing.
|
||||
WHAT_MAX_CHARS = 240
|
||||
|
||||
|
||||
def require_claim(what: str | None) -> None:
|
||||
"""Refuse a `what` that is not one line — before anything is written.
|
||||
|
||||
A refusal rather than a cut, because a cut name reads as complete: the
|
||||
writer would never learn the story went to the wrong field, and the
|
||||
reader would get its first 240 characters as though they were the claim.
|
||||
"""
|
||||
text = (what or "").strip()
|
||||
multiline = "\n" in text
|
||||
if not multiline and len(text) <= WHAT_MAX_CHARS:
|
||||
return
|
||||
shape = "runs over several lines" if multiline else f"is {len(text)} characters"
|
||||
raise ValueError(
|
||||
f"`what` {shape}; it is the lesson's NAME — one claim, one line, at "
|
||||
f"most {WHAT_MAX_CHARS} characters — and every listing and menu prints "
|
||||
"it whole. Nothing was written. Put the claim in `what`, said as the "
|
||||
"sentence someone should come away with, and the incident that taught "
|
||||
"it in `insight`, which is the body and is not bounded."
|
||||
)
|
||||
|
||||
|
||||
def claim_line(what: str | None) -> str:
|
||||
"""A stored name as one menu line, whatever was stored.
|
||||
|
||||
The display half of `require_claim`, for names written before it existed:
|
||||
a guard at the door does not undo a value already stored, and on an
|
||||
install nobody has repaired, those rows would still print their whole
|
||||
story on every menu. Over the bound, the name's first sentence is shown
|
||||
and marked as cut — the record itself is unchanged, and opening it shows
|
||||
the rest.
|
||||
"""
|
||||
text = " ".join((what or "").split())
|
||||
if len(text) <= WHAT_MAX_CHARS:
|
||||
return text
|
||||
first = re.split(r"(?<=[.!?])\s", text, maxsplit=1)[0]
|
||||
if len(first) > WHAT_MAX_CHARS:
|
||||
first = first[:WHAT_MAX_CHARS].rsplit(" ", 1)[0]
|
||||
return first + " …"
|
||||
|
||||
|
||||
def lesson_document(
|
||||
what: str, when_to_apply: str = "", insight: str = "",
|
||||
learned_from: list[int] | None = None,
|
||||
) -> tuple[str, str]:
|
||||
"""The (title, body) a lesson is STORED as.
|
||||
|
||||
One call so the two halves cannot be composed apart. The body's first line
|
||||
carries the trigger; a lesson without it (and without the `data` mirror)
|
||||
would embed as an ordinary note wearing a label, and nothing would report
|
||||
it: the record would look right in every listing and simply never be
|
||||
retrieved at the moment it applies.
|
||||
|
||||
The title is the SUBJECT alone. A stored title carrying the trigger would
|
||||
make every listing, menu and search row show a title that runs to
|
||||
kilobytes. The trigger joins the title at embed time
|
||||
(`embeddings.document_title`, reading `data`).
|
||||
"""
|
||||
return (
|
||||
(what or "").strip(),
|
||||
compose_body(insight, when_to_apply, learned_from),
|
||||
)
|
||||
|
||||
|
||||
def compose_data(
|
||||
what: str, when_to_apply: str = "", learned_from: list[int] | None = None,
|
||||
) -> dict:
|
||||
"""The indexed mirror of the same fields the body renders (0070).
|
||||
|
||||
Written together with the body by the one caller that composes both, so the
|
||||
two can never describe different things — the discipline `compose_data`
|
||||
follows for snippets, and the reason `lesson_trigger` can prefer `data`
|
||||
without checking whether it agrees with the prose.
|
||||
|
||||
Empty values are omitted so the column stays sparse: a lesson with no
|
||||
sources has no `taught_by` key rather than an empty list, which keeps a
|
||||
`?` containment query honest.
|
||||
"""
|
||||
data: dict = {"what": what.strip()} if what and what.strip() else {}
|
||||
trigger = (when_to_apply or "").strip()
|
||||
if trigger:
|
||||
data[TRIGGER_KEY] = trigger
|
||||
sources = normalize_sources(learned_from)
|
||||
if sources:
|
||||
data[SOURCES_KEY] = sources
|
||||
return data
|
||||
|
||||
|
||||
def recompose_data(note) -> dict:
|
||||
"""Rebuild a lesson's `data` mirror from its own title and body.
|
||||
|
||||
For the GENERIC note door. `update_lesson` composes the mirror itself from
|
||||
the merged field set and never needs this; a plain `update_note(body=...)`
|
||||
has no idea the mirror exists and would leave it behind.
|
||||
|
||||
THE COST OF LEAVING IT BEHIND IS HIGHER HERE THAN FOR A SNIPPET. A stale
|
||||
snippet mirror reports the wrong path. A stale lesson mirror reports the
|
||||
wrong TRIGGER — and `lesson_trigger` prefers the mirror, so the lesson goes
|
||||
on being retrieved for the situation it used to name while displaying the
|
||||
one it now names. The trigger is the entire retrieval story, so
|
||||
that is not a degraded record; it is a record that fires at the wrong
|
||||
moment and looks right when it does.
|
||||
|
||||
The body is the authority and the mirror is derived — already this file's
|
||||
rule. This is its enforcement on the path that bypasses `update_lesson`.
|
||||
|
||||
The subject comes back out of the title through `untrigger_title`, the
|
||||
inverse of the join that composed it, rather than by splitting on a
|
||||
separator spelled a second time here.
|
||||
"""
|
||||
from grimoire.services.embeddings import untrigger_title
|
||||
|
||||
body = getattr(note, "body", None) or ""
|
||||
trigger_match = _BODY_TRIGGER_RE.search(body)
|
||||
trigger = trigger_match.group(1).strip() if trigger_match else ""
|
||||
what = untrigger_title(getattr(note, "title", None), trigger)
|
||||
# Sources through the normal read, which already falls back body →
|
||||
# arose_from_id. A body edit that drops the provenance line should drop
|
||||
# the mirror's copy too: the body is the authority, and carrying a value
|
||||
# the reader just deleted is the failure this function exists to prevent.
|
||||
sources_match = _BODY_SOURCES_RE.search(body)
|
||||
sources = (
|
||||
normalize_sources(_ID_RE.findall(sources_match.group(1)))
|
||||
if sources_match else []
|
||||
)
|
||||
return compose_data(what, trigger, sources)
|
||||
|
||||
|
||||
async def create_lesson(
|
||||
user_id: int,
|
||||
*,
|
||||
what: str,
|
||||
when_to_apply: str = "",
|
||||
insight: str = "",
|
||||
learned_from: list[int] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
project_id: int | None = None,
|
||||
):
|
||||
"""Create a lesson note. Returns the created Note.
|
||||
|
||||
The title, the body and the `data` mirror are composed HERE from named
|
||||
parameters rather than asked of the caller. That is the design, not a
|
||||
convenience: a named structured field that a service composes is filled
|
||||
reliably, where a convention an agent must type correctly is not.
|
||||
|
||||
`project_id` is accepted and kept, even though a lesson is reachable from
|
||||
every project. Where it was learned is a fact worth keeping; it
|
||||
simply stops being the limit of where it can be found.
|
||||
"""
|
||||
from grimoire.services import notes as notes_svc
|
||||
|
||||
sources = normalize_sources(learned_from)
|
||||
title, body = lesson_document(what, when_to_apply, insight, sources)
|
||||
return await notes_svc.create_note(
|
||||
user_id,
|
||||
title=title,
|
||||
body=body,
|
||||
note_type=LESSON_NOTE_TYPE,
|
||||
tags=tags,
|
||||
project_id=project_id,
|
||||
# NULL unless there is exactly one source — see `sole_source`.
|
||||
arose_from_id=sole_source(sources),
|
||||
data=compose_data(what, when_to_apply, sources),
|
||||
)
|
||||
|
||||
|
||||
async def get_lesson(user_id: int, lesson_id: int):
|
||||
"""Fetch a lesson by id, or None if it isn't one / isn't readable.
|
||||
|
||||
Share-aware (services/access.py): a fetch by id is an explicit act, so it
|
||||
resolves the caller's full read scope rather than ownership alone —
|
||||
without this, a lesson a search legitimately surfaced could not then be
|
||||
opened.
|
||||
"""
|
||||
from grimoire.services import notes as notes_svc
|
||||
|
||||
result = await notes_svc.get_note_for_user(user_id, lesson_id)
|
||||
if result is None:
|
||||
return None
|
||||
note, _permission = result
|
||||
if note.note_type != LESSON_NOTE_TYPE or note.deleted_at is not None:
|
||||
return None
|
||||
return note
|
||||
|
||||
|
||||
def lesson_to_dict(note) -> dict:
|
||||
"""A lesson as either door returns it — the composed fields read back out,
|
||||
not the raw row, so a caller sees the same vocabulary it wrote with.
|
||||
|
||||
In the SERVICE rather than in each door, on the `snippet_to_dict`
|
||||
precedent: the REST route feeds the web UI and the MCP tools feed an
|
||||
agent, and a shape spelled once per door is a shape that answers the two
|
||||
of them differently the first time a field is added.
|
||||
"""
|
||||
return {
|
||||
"id": note.id,
|
||||
"title": note.title,
|
||||
"body": note.body,
|
||||
# The composed vocabulary, not the storage: a caller that wrote
|
||||
# `when_to_apply` reads `when_to_apply` back.
|
||||
"what": (note.data or {}).get("what", "") if isinstance(note.data, dict) else "",
|
||||
"when_to_apply": lesson_trigger(note),
|
||||
"learned_from": lesson_sources(note),
|
||||
"insight": lesson_insight(note),
|
||||
"tags": list(note.tags or []),
|
||||
"project_id": note.project_id,
|
||||
"note_type": note.note_type,
|
||||
"created_at": note.created_at.isoformat() if note.created_at else None,
|
||||
"updated_at": note.updated_at.isoformat() if note.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
async def source_records(user_id: int, ids: list[int]) -> list[dict]:
|
||||
"""The records that taught a lesson, resolved to something linkable.
|
||||
|
||||
`learned_from` is a list of bare ids, which is right for storage and
|
||||
useless on a page: "#4181" tells a reader nothing about whether it is
|
||||
worth opening. This resolves each to its title and kind so the UI can
|
||||
label the link, and so a source that has been deleted simply drops out
|
||||
rather than rendering a link to nothing.
|
||||
|
||||
One query for the whole list, not one per id — a lesson with six sources
|
||||
would otherwise be six round trips to draw one panel.
|
||||
|
||||
Share-aware (services/access.py). Order follows `ids`, because that order is the
|
||||
writer's: the first source is the one they reached for first.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.services.access import readable_notes_clause
|
||||
|
||||
wanted = normalize_sources(ids)
|
||||
if not wanted:
|
||||
return []
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Note)
|
||||
.where(Note.id.in_(wanted))
|
||||
.where(Note.deleted_at.is_(None))
|
||||
.where(readable_notes_clause(user_id))
|
||||
)
|
||||
found = {n.id: n for n in result.scalars().all()}
|
||||
return [
|
||||
{
|
||||
"id": n.id,
|
||||
"title": n.title,
|
||||
"note_type": n.note_type,
|
||||
"is_task": n.status is not None,
|
||||
"task_kind": n.task_kind,
|
||||
"status": n.status,
|
||||
}
|
||||
for i in wanted
|
||||
if (n := found.get(i)) is not None
|
||||
]
|
||||
|
||||
|
||||
async def lessons_taught_by(user_id: int, record_id: int, limit: int = 20):
|
||||
"""The lessons drawn FROM one record — the reverse of `learned_from`.
|
||||
|
||||
THE DIRECTION THAT GETS FORGOTTEN, and arguably the more useful one: a
|
||||
reader opening an old issue wants to know what was learned from it, and
|
||||
without this the relation is only navigable from the lesson's side. A
|
||||
record that taught something should say so on its own page.
|
||||
|
||||
Queried through `data[SOURCES_KEY]` rather than by scanning bodies: the
|
||||
mirror is JSONB with a GIN index, which is the whole reason the list
|
||||
lives there. `path_exists` is the same dialect the snippet location
|
||||
lookup uses, so both reverse lookups read the index the same way.
|
||||
|
||||
Share-aware via `readable_notes_clause`: this renders beside a
|
||||
record the caller can already see, and a lesson someone shared with them
|
||||
belongs in that list exactly as their own does.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.services.access import readable_notes_clause
|
||||
|
||||
try:
|
||||
wanted = int(record_id)
|
||||
except (TypeError, ValueError):
|
||||
return []
|
||||
if wanted <= 0:
|
||||
return []
|
||||
|
||||
# The id is an int we just validated, never caller text, so it cannot
|
||||
# break out of the expression — the same guarantee `location_jsonpath`
|
||||
# gets from JSON-quoting its values.
|
||||
jsonpath = f"$.{SOURCES_KEY}[*] ? (@ == {wanted})"
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Note)
|
||||
.where(Note.note_type == LESSON_NOTE_TYPE)
|
||||
.where(Note.deleted_at.is_(None))
|
||||
.where(Note.data.path_exists(jsonpath))
|
||||
.where(readable_notes_clause(user_id))
|
||||
.order_by(Note.updated_at.desc())
|
||||
.limit(max(1, min(limit, 100)))
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def update_lesson(
|
||||
user_id: int,
|
||||
lesson_id: int,
|
||||
*,
|
||||
what: str | None = None,
|
||||
when_to_apply: str | None = None,
|
||||
insight: str | None = None,
|
||||
learned_from: list[int] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
):
|
||||
"""Update a lesson, re-composing title, body and mirror from the merged
|
||||
fields. Returns the updated Note, or None if it isn't a readable lesson.
|
||||
|
||||
READ-MODIFY-WRITE over the whole record rather than patching one half.
|
||||
The three fields are not independent: the trigger appears in the title AND
|
||||
at the head of the body, so editing it in place would need two edits that
|
||||
a caller could do one of. Re-composing from the merged values means a
|
||||
partial update cannot leave the halves disagreeing — which, because the
|
||||
document is what ranks, would be a lesson that still reads correctly and
|
||||
quietly stops being retrievable.
|
||||
"""
|
||||
from grimoire.services import notes as notes_svc
|
||||
|
||||
note = await get_lesson(user_id, lesson_id)
|
||||
if note is None:
|
||||
return None
|
||||
|
||||
current = note.data if isinstance(note.data, dict) else {}
|
||||
merged_what = current.get("what") or "" if what is None else what
|
||||
merged_trigger = lesson_trigger(note) if when_to_apply is None else when_to_apply
|
||||
merged_sources = (
|
||||
lesson_sources(note) if learned_from is None
|
||||
else normalize_sources(learned_from)
|
||||
)
|
||||
if insight is None:
|
||||
insight = lesson_insight(note)
|
||||
|
||||
title, body = lesson_document(
|
||||
merged_what, merged_trigger, insight, merged_sources,
|
||||
)
|
||||
fields: dict = {
|
||||
"title": title,
|
||||
"body": body,
|
||||
"arose_from_id": sole_source(merged_sources),
|
||||
"data": compose_data(merged_what, merged_trigger, merged_sources),
|
||||
}
|
||||
if tags is not None:
|
||||
fields["tags"] = tags
|
||||
return await notes_svc.update_note(user_id, lesson_id, **fields)
|
||||
|
||||
|
||||
def lesson_insight(note) -> str:
|
||||
"""The authored insight alone — the body with what `compose_body` added removed.
|
||||
|
||||
An update that keeps the insight hands it back to `compose_body`, which
|
||||
re-adds the trigger and provenance. So this has to remove EXACTLY what was
|
||||
added; anything it leaves behind is re-prepended and the body grows by that
|
||||
much on every edit.
|
||||
|
||||
TAKES THE NOTE, NOT THE BODY. Filtering out lines matching
|
||||
`_BODY_TRIGGER_RE` is not enough: that pattern is `re.M` and matches ONE
|
||||
LINE, while `when_to_apply` is free text and routinely several paragraphs
|
||||
— `compose_body` writes all of it after the marker. Paragraphs 2..n would
|
||||
come back as part of "the insight", the full trigger would be prepended
|
||||
again, and the record would gain (n-1) paragraphs per update. Silent, and
|
||||
compounding.
|
||||
|
||||
What makes an exact strip possible is that the trigger is stored: the
|
||||
mirror in `data` holds its full text, so `composed` below is
|
||||
byte-for-byte what `compose_body` put at the head, and removing it is
|
||||
subtraction rather than pattern-matching.
|
||||
|
||||
The fallback is not dead code. A row whose mirror is missing degrades to
|
||||
line-wise removal, which is right for a one-paragraph trigger and is the
|
||||
best that can be done without knowing where the trigger ends — there is
|
||||
no marker for its last line.
|
||||
|
||||
The provenance line needs no such care: `compose_body` builds it by
|
||||
joining ids with ", ", so it cannot contain a newline.
|
||||
"""
|
||||
body = getattr(note, "body", None) or ""
|
||||
trigger = lesson_trigger(note).strip()
|
||||
head = body.lstrip("\n")
|
||||
composed = f"**When to apply:** {trigger}" if trigger else ""
|
||||
if composed and head.startswith(composed):
|
||||
lines = head[len(composed):].splitlines()
|
||||
else:
|
||||
lines = [ln for ln in body.splitlines() if not _BODY_TRIGGER_RE.match(ln)]
|
||||
kept = [ln for ln in lines if not _BODY_SOURCES_RE.match(ln)]
|
||||
return "\n".join(kept).strip()
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Application logging service for audit, usage, and error events."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import traceback as tb_module
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete, func, select, text
|
||||
|
||||
from grimoire.config import Config
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_retention_task: asyncio.Task | None = None
|
||||
|
||||
|
||||
async def log_audit(
|
||||
action: str,
|
||||
user_id: int | None = None,
|
||||
username: str | None = None,
|
||||
ip_address: str | None = None,
|
||||
details: dict | None = None,
|
||||
) -> None:
|
||||
async with async_session() as session:
|
||||
log = AppLog(
|
||||
category="audit",
|
||||
user_id=user_id,
|
||||
username=username,
|
||||
action=action,
|
||||
ip_address=ip_address,
|
||||
details=json.dumps(details) if details else None,
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def log_usage(
|
||||
user_id: int | None = None,
|
||||
username: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
method: str | None = None,
|
||||
status_code: int | None = None,
|
||||
duration_ms: float | None = None,
|
||||
) -> None:
|
||||
async with async_session() as session:
|
||||
log = AppLog(
|
||||
category="usage",
|
||||
user_id=user_id,
|
||||
username=username,
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
status_code=status_code,
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def log_error(
|
||||
user_id: int | None = None,
|
||||
username: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
method: str | None = None,
|
||||
error_type: str | None = None,
|
||||
error_message: str | None = None,
|
||||
traceback: str | None = None,
|
||||
) -> None:
|
||||
details = {}
|
||||
if error_type:
|
||||
details["error_type"] = error_type
|
||||
if error_message:
|
||||
details["error_message"] = error_message
|
||||
if traceback:
|
||||
details["traceback"] = traceback
|
||||
|
||||
async with async_session() as session:
|
||||
log = AppLog(
|
||||
category="error",
|
||||
user_id=user_id,
|
||||
username=username,
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
details=json.dumps(details) if details else None,
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
|
||||
|
||||
def parse_filter_datetime(value: str | None, *, end_of_day: bool = False) -> datetime | None:
|
||||
"""An ISO date/datetime from a query string as an AWARE UTC datetime.
|
||||
|
||||
The admin log filters arrive as raw `request.args` strings. Compared
|
||||
straight against `AppLog.created_at`, they fail: asyncpg binds a str as
|
||||
VARCHAR and Postgres has no `timestamptz >= text` operator, so either date
|
||||
filter would raise.
|
||||
|
||||
Returns None for anything unparseable: a malformed filter should narrow
|
||||
nothing rather than 500 the log viewer.
|
||||
|
||||
`end_of_day` matters for the upper bound. A bare "2026-07-30" parses to
|
||||
midnight, so `created_at <= that` would exclude the whole of the day the
|
||||
user asked for — the one day they most likely wanted. With this flag a
|
||||
date-only value is pushed to the last microsecond of that day; a value that
|
||||
already carries a time is left exactly as given.
|
||||
"""
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
if end_of_day and len(raw) == 10: # date-only, no time component
|
||||
parsed = parsed.replace(hour=23, minute=59, second=59, microsecond=999999)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed
|
||||
|
||||
|
||||
async def get_logs(
|
||||
category: str | None = None,
|
||||
user_id: int | None = None,
|
||||
search: str | None = None,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
async with async_session() as session:
|
||||
query = select(AppLog)
|
||||
count_query = select(func.count(AppLog.id))
|
||||
|
||||
if category:
|
||||
query = query.where(AppLog.category == category)
|
||||
count_query = count_query.where(AppLog.category == category)
|
||||
if user_id is not None:
|
||||
query = query.where(AppLog.user_id == user_id)
|
||||
count_query = count_query.where(AppLog.user_id == user_id)
|
||||
if search:
|
||||
pattern = f"%{search}%"
|
||||
search_filter = (
|
||||
AppLog.action.ilike(pattern)
|
||||
| AppLog.endpoint.ilike(pattern)
|
||||
| AppLog.username.ilike(pattern)
|
||||
| AppLog.details.ilike(pattern)
|
||||
)
|
||||
query = query.where(search_filter)
|
||||
count_query = count_query.where(search_filter)
|
||||
start = parse_filter_datetime(date_from)
|
||||
end = parse_filter_datetime(date_to, end_of_day=True)
|
||||
if start:
|
||||
query = query.where(AppLog.created_at >= start)
|
||||
count_query = count_query.where(AppLog.created_at >= start)
|
||||
if end:
|
||||
query = query.where(AppLog.created_at <= end)
|
||||
count_query = count_query.where(AppLog.created_at <= end)
|
||||
|
||||
total = (await session.execute(count_query)).scalar() or 0
|
||||
|
||||
query = query.order_by(AppLog.created_at.desc()).limit(limit).offset(offset)
|
||||
result = await session.execute(query)
|
||||
logs = [row.to_dict() for row in result.scalars().all()]
|
||||
|
||||
return logs, total
|
||||
|
||||
|
||||
async def get_log_stats() -> dict:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
text(
|
||||
"SELECT category, COUNT(*) as count FROM app_logs GROUP BY category"
|
||||
)
|
||||
)
|
||||
stats = {row.category: row.count for row in result}
|
||||
return {
|
||||
"audit": stats.get("audit", 0),
|
||||
"usage": stats.get("usage", 0),
|
||||
"error": stats.get("error", 0),
|
||||
"total": sum(stats.values()),
|
||||
}
|
||||
|
||||
|
||||
async def delete_old_logs(retention_days: int) -> int:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
delete(AppLog).where(AppLog.created_at < cutoff)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
|
||||
|
||||
async def _retention_tick() -> None:
|
||||
deleted = await delete_old_logs(Config.LOG_RETENTION_DAYS)
|
||||
if deleted:
|
||||
logger.info("Log retention: deleted %d old log entries", deleted)
|
||||
|
||||
|
||||
def start_log_retention_loop() -> None:
|
||||
global _retention_task
|
||||
if _retention_task is None or _retention_task.done():
|
||||
from grimoire.services.background import start_periodic
|
||||
_retention_task = start_periodic(3600, _retention_tick, label="log_retention") # hourly
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Milestone management service."""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.milestone import Milestone
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.services import access as access_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The statuses that make a step OPEN work, defined here because this is the
|
||||
# lower layer: services/placement.py imports it rather than restating it. Two
|
||||
# surfaces that both answer "what is next" and disagree about what counts as
|
||||
# next is worse than one of them staying silent.
|
||||
OPEN_STEP_STATUSES = ("todo", "in_progress")
|
||||
|
||||
|
||||
def embed_milestone(milestone: Milestone) -> None:
|
||||
"""Refresh a milestone's vectors, fire-and-forget.
|
||||
|
||||
At the service, so every path that writes a milestone gets it: a record
|
||||
written through a door that forgot the call stays out of search until a
|
||||
restart. Exceptions are swallowed because
|
||||
a milestone that saved must not fail on its index refresh; no running loop
|
||||
(a script, a unit test) is ordinary. A delete racing the refresh wins: the
|
||||
upsert claims the milestone's row first.
|
||||
"""
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from grimoire.services.embeddings import upsert_milestone_embedding
|
||||
|
||||
asyncio.create_task(upsert_milestone_embedding(
|
||||
milestone.id, milestone.title, milestone.description, milestone.body,
|
||||
))
|
||||
except RuntimeError:
|
||||
pass
|
||||
except Exception: # noqa: BLE001 - never let indexing break a write
|
||||
logger.exception("embedding refresh failed for milestone %s", milestone.id)
|
||||
|
||||
|
||||
async def create_milestone(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
title: str,
|
||||
description: str | None = None,
|
||||
body: str | None = None,
|
||||
order_index: int = 0,
|
||||
status: str = "active",
|
||||
) -> Milestone:
|
||||
async with async_session() as session:
|
||||
milestone = Milestone(
|
||||
user_id=user_id,
|
||||
project_id=project_id,
|
||||
title=title,
|
||||
description=description,
|
||||
body=body,
|
||||
status=status,
|
||||
order_index=order_index,
|
||||
)
|
||||
session.add(milestone)
|
||||
await session.commit()
|
||||
await session.refresh(milestone)
|
||||
embed_milestone(milestone)
|
||||
return milestone
|
||||
|
||||
|
||||
async def get_milestone(user_id: int, milestone_id: int) -> Milestone | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.id == milestone_id, Milestone.user_id == user_id,
|
||||
Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def titles_for(milestone_ids: set[int]) -> dict[int, str]:
|
||||
"""{id: title} for the given milestones, for rows that name where a record sits.
|
||||
|
||||
No ownership filter, deliberately: the callers are listings whose rows the
|
||||
caller could already read, and a milestone title is part of "where does
|
||||
this record sit". Filtering here would blank the placement of a shared
|
||||
task in someone else's plan while still showing the task.
|
||||
"""
|
||||
ids = {i for i in milestone_ids if i}
|
||||
if not ids:
|
||||
return {}
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(Milestone.id, Milestone.title).where(
|
||||
Milestone.id.in_(ids), Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)).all()
|
||||
return {mid: title for mid, title in rows}
|
||||
|
||||
|
||||
async def get_milestone_in_project(project_id: int, milestone_id: int) -> Milestone | None:
|
||||
"""Fetch a milestone by id within a project, without a user_id ownership check.
|
||||
Callers must verify project access separately before using this."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.id == milestone_id,
|
||||
Milestone.project_id == project_id,
|
||||
Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def get_milestone_by_title(user_id: int, project_id: int, title: str) -> Milestone | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.user_id == user_id,
|
||||
Milestone.project_id == project_id,
|
||||
func.lower(Milestone.title) == func.lower(title.strip()),
|
||||
).limit(1)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def find_milestone_by_title(user_id: int, title: str) -> Milestone | None:
|
||||
"""Find a milestone by title across ALL projects for this user (case-insensitive)."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.user_id == user_id,
|
||||
func.lower(Milestone.title) == func.lower(title.strip()),
|
||||
).order_by(Milestone.id).limit(1)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def get_or_create_milestone(user_id: int, project_id: int, title: str) -> Milestone:
|
||||
milestone = await get_milestone_by_title(user_id, project_id, title)
|
||||
if milestone:
|
||||
return milestone
|
||||
return await create_milestone(user_id, project_id, title=title)
|
||||
|
||||
|
||||
async def list_milestones(
|
||||
user_id: int, project_id: int, status: str | None = None
|
||||
) -> list[Milestone]:
|
||||
async with async_session() as session:
|
||||
query = select(Milestone).where(
|
||||
Milestone.user_id == user_id,
|
||||
Milestone.project_id == project_id,
|
||||
Milestone.deleted_at.is_(None),
|
||||
)
|
||||
if status:
|
||||
query = query.where(Milestone.status == status)
|
||||
query = query.order_by(Milestone.order_index.asc(), Milestone.created_at.asc())
|
||||
result = await session.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def update_milestone(user_id: int, milestone_id: int, **fields: object) -> Milestone | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.id == milestone_id, Milestone.user_id == user_id,
|
||||
Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
milestone = result.scalars().first()
|
||||
if milestone is None:
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if hasattr(milestone, key):
|
||||
setattr(milestone, key, value)
|
||||
milestone.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(milestone)
|
||||
if {"title", "description", "body"} & set(fields):
|
||||
embed_milestone(milestone)
|
||||
return milestone
|
||||
|
||||
|
||||
async def delete_milestone(user_id: int, milestone_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Milestone).where(Milestone.id == milestone_id, Milestone.user_id == user_id)
|
||||
)
|
||||
milestone = result.scalars().first()
|
||||
if milestone is None:
|
||||
return False
|
||||
await session.delete(milestone)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def get_milestone_progress(milestone_id: int) -> dict:
|
||||
"""Return task completion stats for a milestone."""
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(Note.status, func.count(Note.id))
|
||||
.where(
|
||||
Note.milestone_id == milestone_id, Note.status.isnot(None),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
.group_by(Note.status)
|
||||
)
|
||||
status_counts: dict[str, int] = {}
|
||||
for status, count in rows.fetchall():
|
||||
status_counts[status] = count
|
||||
|
||||
# Same rule as the batch path, computed in one place so the two cannot
|
||||
# drift on the cancelled-exclusion.
|
||||
return _progress_from_counts(status_counts)
|
||||
|
||||
|
||||
def _progress_from_counts(status_counts: dict[str, int]) -> dict:
|
||||
"""The progress shape, computed from already-fetched counts.
|
||||
|
||||
Split out of get_milestone_progress so the batch path can reuse the rule
|
||||
rather than restate it — the cancelled-exclusion below is easy to get
|
||||
subtly different in a second copy, and then two screens disagree about
|
||||
whether a milestone is finished.
|
||||
"""
|
||||
total = sum(status_counts.values())
|
||||
cancelled = status_counts.get("cancelled", 0)
|
||||
completed = status_counts.get("done", 0)
|
||||
# Cancelled tasks are resolved work, not pending — excluded from the
|
||||
# denominator so a milestone whose only open task was cancelled reaches
|
||||
# 100% instead of stalling.
|
||||
active_total = total - cancelled
|
||||
return {
|
||||
"total": total,
|
||||
"completed": completed,
|
||||
"pct": round(completed / active_total * 100, 1) if active_total > 0 else 0.0,
|
||||
"status_counts": {
|
||||
"todo": status_counts.get("todo", 0),
|
||||
"in_progress": status_counts.get("in_progress", 0),
|
||||
"done": status_counts.get("done", 0),
|
||||
"cancelled": cancelled,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def get_project_milestone_summaries(
|
||||
user_id: int, project_ids: list[int]
|
||||
) -> dict[int, list[dict]]:
|
||||
"""Milestone summaries for MANY projects in three queries total.
|
||||
|
||||
The per-project version below is a nested fan-out: one query to list a
|
||||
project's milestones, then one more per milestone for its progress. Called
|
||||
for many projects concurrently it asks for far more pooled connections than
|
||||
the pool holds, and every one of them waits out the checkout timeout. This
|
||||
does the same work in a fixed number of queries and one
|
||||
session: the milestones, their step counts, and their open steps.
|
||||
|
||||
Each row carries `next_step` — the earliest open step, or None.
|
||||
That is NOT the same question `services/placement.py` answers: placement
|
||||
knows which step you are on and names the next one AFTER it, while a
|
||||
listing has no current step, so the earliest open one is the whole answer.
|
||||
The two share `OPEN_STEP_STATUSES` and the creation ordering so they can
|
||||
never disagree about which steps are candidates.
|
||||
"""
|
||||
if not project_ids:
|
||||
return {}
|
||||
|
||||
async with async_session() as session:
|
||||
milestones = list((await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.user_id == user_id,
|
||||
Milestone.project_id.in_(project_ids),
|
||||
Milestone.deleted_at.is_(None),
|
||||
).order_by(Milestone.order_index.asc(), Milestone.created_at.asc())
|
||||
)).scalars().all())
|
||||
|
||||
counts: dict[int, dict[str, int]] = {}
|
||||
step_touched: dict[int, datetime] = {}
|
||||
next_step: dict[int, dict] = {}
|
||||
if milestones:
|
||||
milestone_ids = [m.id for m in milestones]
|
||||
# Both step queries below take the SAME visibility clause.
|
||||
# They have to: `next_step` names a step and the counts beside it
|
||||
# say how many there are, so a row that could name a step its own
|
||||
# progress excludes would be reporting two different milestones.
|
||||
readable = access_svc.readable_notes_clause(user_id)
|
||||
rows = await session.execute(
|
||||
select(
|
||||
Note.milestone_id, Note.status, func.count(Note.id),
|
||||
func.max(Note.updated_at),
|
||||
)
|
||||
.where(
|
||||
Note.milestone_id.in_(milestone_ids),
|
||||
Note.status.isnot(None),
|
||||
Note.deleted_at.is_(None),
|
||||
readable,
|
||||
)
|
||||
.group_by(Note.milestone_id, Note.status)
|
||||
)
|
||||
for milestone_id, status, count, latest in rows.fetchall():
|
||||
counts.setdefault(milestone_id, {})[status] = count
|
||||
if latest and (milestone_id not in step_touched
|
||||
or latest > step_touched[milestone_id]):
|
||||
step_touched[milestone_id] = latest
|
||||
|
||||
# ONE query for every milestone in the batch, not one per row.
|
||||
# Fanned out per milestone, this listing drains the connection
|
||||
# pool; a third flat query keeps the cost
|
||||
# constant in the number of plans. Ordered the way
|
||||
# services/placement.py orders steps — creation order, the order a
|
||||
# plan is written and a batch create inserts — so the first row per
|
||||
# milestone IS its next open step.
|
||||
open_rows = await session.execute(
|
||||
select(Note.milestone_id, Note.id, Note.title, Note.status)
|
||||
.where(
|
||||
Note.milestone_id.in_(milestone_ids),
|
||||
Note.status.in_(OPEN_STEP_STATUSES),
|
||||
Note.deleted_at.is_(None),
|
||||
readable,
|
||||
)
|
||||
.order_by(Note.created_at.asc(), Note.id.asc())
|
||||
)
|
||||
for milestone_id, note_id, title, status in open_rows.fetchall():
|
||||
next_step.setdefault(
|
||||
milestone_id, {"id": note_id, "title": title, "status": status},
|
||||
)
|
||||
|
||||
out: dict[int, list[dict]] = {pid: [] for pid in project_ids}
|
||||
for m in milestones:
|
||||
entry = m.to_dict()
|
||||
entry.update(_progress_from_counts(counts.get(m.id, {})))
|
||||
# A milestone's own updated_at doesn't move when its steps do, so a
|
||||
# plan whose steps closed today would read as untouched since it was
|
||||
# written. Touched is the later of the two.
|
||||
touched = [t for t in (m.updated_at, step_touched.get(m.id)) if t]
|
||||
entry["last_touched_at"] = max(touched).isoformat() if touched else None
|
||||
# Always present, None included. "8 of 9 done" tells a reader there is
|
||||
# an open step and not WHICH, and a gap that shape gets filled from
|
||||
# whatever id is nearest to hand — a retrieval hint carries an id and a
|
||||
# title and no status, so a done step can be reported as the open one.
|
||||
# A listing that names it leaves nothing to guess.
|
||||
entry["next_step"] = next_step.get(m.id)
|
||||
out.setdefault(m.project_id, []).append(entry)
|
||||
return out
|
||||
|
||||
|
||||
async def get_project_milestone_summary(user_id: int, project_id: int) -> list[dict]:
|
||||
"""Ordered milestones with progress — the one-project view of
|
||||
get_project_milestone_summaries (two queries, not N+1)."""
|
||||
return (await get_project_milestone_summaries(user_id, [project_id])).get(project_id, [])
|
||||
|
||||
|
||||
# What a milestone LISTING needs: enough to say what each plan is, how far
|
||||
# along it is, and which step is next. The plan itself (`body`) is
|
||||
# get_milestone's job: on a project with a few dozen milestones, plan bodies
|
||||
# would push enter_project past what an MCP client accepts as a tool result,
|
||||
# and the session handshake would arrive as a file to page through.
|
||||
# user_id / project_id / timestamps repeat what the caller already knows.
|
||||
_BRIEF_FIELDS = (
|
||||
"id", "title", "description", "status", "order_index",
|
||||
"total", "completed", "pct", "status_counts", "next_step",
|
||||
)
|
||||
|
||||
|
||||
def brief_milestone_summary(
|
||||
rows: list[dict], *, limit: int | None = None,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Trim summary rows to the listing fields, optionally keeping only the
|
||||
most recently touched.
|
||||
|
||||
With `limit`, keeps the N rows with the latest `last_touched_at`, whatever
|
||||
their status, most recent first: a handshake says what was worked on
|
||||
lately, and "active" alone doesn't (a plan can sit active for months).
|
||||
Without it, every row stays in its original order. Returns (rows, omitted).
|
||||
"""
|
||||
kept = rows
|
||||
if limit is not None:
|
||||
kept = sorted(
|
||||
rows, key=lambda r: r.get("last_touched_at") or "", reverse=True,
|
||||
)[:limit]
|
||||
brief = [{k: r[k] for k in _BRIEF_FIELDS if k in r} for r in kept]
|
||||
return brief, len(rows) - len(kept)
|
||||
|
||||
|
||||
def unplanned_milestones(
|
||||
rows: list[dict], *, exclude_ids: set[int] = frozenset(), limit: int | None = None,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Active milestones with no steps yet, as (rows, omitted).
|
||||
|
||||
A plan written as a milestone with a description and no steps is open work
|
||||
that nothing else names. It is never "touched" — touching is a step
|
||||
changing — so the recency list that brief_milestone_summary(limit=) builds
|
||||
can never reach it, and progress reads 0% either way. Unlisted, a roadmap
|
||||
written that way gets every later plan opened as a new milestone beside the
|
||||
one that already described it.
|
||||
|
||||
`exclude_ids` drops milestones a caller already listed. Kept in roadmap
|
||||
order (order_index, then creation), the order they were written in.
|
||||
Rows are id, title and description: what a reader needs to recognise the
|
||||
plan, and not its body, which get_milestone reads.
|
||||
"""
|
||||
found = [
|
||||
{"id": r["id"], "title": r.get("title"), "description": r.get("description")}
|
||||
for r in rows
|
||||
if r.get("status") == "active" and not r.get("total") and r["id"] not in exclude_ids
|
||||
]
|
||||
kept = found if limit is None else found[:limit]
|
||||
return kept, len(found) - len(kept)
|
||||
@@ -0,0 +1,481 @@
|
||||
"""Which actions reach which moment: shipped defaults plus each install's own.
|
||||
|
||||
The catalog (`services/moments.py`) says what the moments ARE. This module says
|
||||
how the work gets there. One call can reach several moments: `kubectl apply`
|
||||
is a run, a deliver and a reach outside the workspace all at once, and a rule
|
||||
mounted on any of them should arrive.
|
||||
|
||||
AN ACTION is a tool plus an optional `match`:
|
||||
|
||||
- `match` empty — every call of that tool (`Edit` → work.change).
|
||||
- A tool that runs a command (its input carries `command`) — how the command
|
||||
starts, tested against each segment of a compound line, so
|
||||
`cd app && make ship` reaches what `make ship` reaches. Leading `VAR=value`
|
||||
assignments are skipped; a word boundary is required, so `git push` does
|
||||
not match `git pushd`.
|
||||
- Any other tool — `field=value` pairs, comma-separated, all of which must
|
||||
hold (`update_task` with `status=done`).
|
||||
|
||||
Tool names are compared without any MCP server prefix and without case:
|
||||
`mcp__plugin_x__update_task` and `update_task` are one tool, because what an
|
||||
install called its server is not something a mapping should depend on.
|
||||
|
||||
WHY THE DEFAULTS ARE CODE AND THE CORRECTIONS ARE ROWS
|
||||
|
||||
The defaults cover the actions every install shares — the harness's own tools,
|
||||
Grimoire's own tools, the commonest command shapes. They ship, and improve, with
|
||||
the product. What no default can know is how one operator works, so an install
|
||||
ADDS mappings and REMOVES defaults that misfire for it, in-session, through
|
||||
`map_action` / `unmap_action`. A removal is stored rather than applied to the
|
||||
defaults so it survives an upgrade that ships the same default again.
|
||||
|
||||
The concrete commands below are examples of reaching a moment, which is why
|
||||
they may name particular tools when the moments themselves may not.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.moment_mapping import MomentMapping
|
||||
from grimoire.services import moments as catalog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ADD, REMOVE = "add", "remove"
|
||||
EFFECTS = (ADD, REMOVE)
|
||||
DEFAULT, INSTALL = "default", "install"
|
||||
|
||||
# The input field that makes a tool a command-runner, and the tools known to
|
||||
# be one. The field drives MATCHING (any tool whose call carries a command is
|
||||
# matched by prefix); the names drive VALIDATION, so a prefix-style match on a
|
||||
# tool that takes no command is refused rather than stored to never fire.
|
||||
COMMAND_FIELD = "command"
|
||||
COMMAND_TOOLS = frozenset({"bash"})
|
||||
|
||||
# The harness's skill loader: loading a procedure reaches `skill.<its name>`,
|
||||
# derived from the call rather than listed, since the names are the
|
||||
# procedures' own.
|
||||
SKILL_TOOL, SKILL_FIELD = "skill", "skill"
|
||||
|
||||
# A procedure also reaches the moment it is FOR: loading the
|
||||
# reporting procedure is a report as surely as the reply that follows it.
|
||||
#
|
||||
# The product's own skills declare theirs in their SKILL.md frontmatter
|
||||
# (`metadata: moments:`), and the declaration ships HERE as defaults, because
|
||||
# the server never sees the plugin's files. `test_skill_moments` holds the two
|
||||
# together. The plugin's name qualifies the skill, as the harness names it.
|
||||
BUNDLED_SKILL_PLUGIN = "scribe"
|
||||
BUNDLED_SKILL_MOMENTS: dict[str, tuple[str, ...]] = {
|
||||
"using-scribe": ("session.start",),
|
||||
"brainstorming": ("work.plan",),
|
||||
"writing-plans": ("work.plan",),
|
||||
"reusing-code": ("work.change",),
|
||||
"systematic-debugging": ("work.debug",),
|
||||
"verification": ("work.verify",),
|
||||
"shape-accounting": ("work.record",),
|
||||
"reporting-back": ("reply.report",),
|
||||
"family-canon": ("work.record",),
|
||||
}
|
||||
|
||||
# A stored process arrives as the skill `scribe-proc-<slug>`
|
||||
# (scribe_sync_processes.sh). Its moments are the process's own field, read
|
||||
# when it loads — not copied into the stub, which is written once a session
|
||||
# and would go on firing a moment its process no longer declares.
|
||||
PROCESS_SKILL_PREFIX = "scribe-proc-"
|
||||
PROCESS_MOMENTS_FIELD = "moments"
|
||||
PROCESS = "process"
|
||||
|
||||
_SEGMENT_SPLIT = re.compile(r"&&|\|\||[;|\n]")
|
||||
_ENV_ASSIGN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=\S*\s+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Action:
|
||||
tool: str
|
||||
match: str
|
||||
moment: str
|
||||
|
||||
|
||||
def _defaults() -> tuple[Action, ...]:
|
||||
out: list[Action] = []
|
||||
|
||||
def on(moment: str, tool: str, *matches: str) -> None:
|
||||
for m in matches or ("",):
|
||||
out.append(Action(tool, m, moment))
|
||||
|
||||
# The harness's own tools.
|
||||
on("work.change", "Edit")
|
||||
on("work.change", "Write")
|
||||
on("work.change", "MultiEdit")
|
||||
on("work.change", "NotebookEdit")
|
||||
on("work.run", "Bash")
|
||||
on("work.delegate", "Task")
|
||||
on("work.delegate", "Agent")
|
||||
on("work.plan", "EnterPlanMode")
|
||||
on("work.plan", "ExitPlanMode")
|
||||
on("reply.ask", "AskUserQuestion")
|
||||
for skill, reached in BUNDLED_SKILL_MOMENTS.items():
|
||||
for moment in reached:
|
||||
on(moment, "Skill", f"{SKILL_FIELD}={BUNDLED_SKILL_PLUGIN}:{skill}")
|
||||
|
||||
# Grimoire's own tools — every install has these, so their moments ship.
|
||||
on("work.start", "update_task", "status=in_progress")
|
||||
on("work.finish", "update_task", "status=done", "status=cancelled")
|
||||
on("work.finish", "update_milestone", "status=done")
|
||||
on("work.plan", "start_planning")
|
||||
on("work.plan", "create_milestone")
|
||||
for tool in ("create_note", "create_lesson", "create_rule",
|
||||
"create_project_rule", "create_preference", "create_snippet",
|
||||
"create_process"):
|
||||
on("work.record", tool)
|
||||
|
||||
# The commonest command shapes. An install's own (`make ship`,
|
||||
# `./deploy.sh`) are what map_action is for.
|
||||
on("work.deliver", "Bash",
|
||||
"git push", "git merge", "gh pr merge", "gh release create",
|
||||
"docker push", "kubectl apply", "helm install", "helm upgrade",
|
||||
"terraform apply", "npm publish", "twine upload", "cargo publish")
|
||||
on("work.verify", "Bash",
|
||||
"pytest", "python -m pytest", "npm test", "npm run test", "yarn test",
|
||||
"pnpm test", "go test", "go vet", "cargo test", "make test",
|
||||
"make check", "ruff check", "mypy", "terraform plan",
|
||||
"terraform validate")
|
||||
on("env.reach", "Bash",
|
||||
"ssh", "scp", "curl", "wget", "kubectl", "helm")
|
||||
return tuple(out)
|
||||
|
||||
|
||||
DEFAULT_ACTIONS: tuple[Action, ...] = _defaults()
|
||||
|
||||
|
||||
# ── matching ──────────────────────────────────────────────────────────────
|
||||
|
||||
def tool_key(name: str) -> str:
|
||||
"""The tool's bare, lowercased name: no MCP server prefix."""
|
||||
clean = (name or "").strip()
|
||||
if clean.startswith("mcp__"):
|
||||
clean = clean.rsplit("__", 1)[-1]
|
||||
return clean.lower()
|
||||
|
||||
|
||||
def _squash(text: str) -> str:
|
||||
return " ".join((text or "").split())
|
||||
|
||||
|
||||
def _segments(command: str) -> list[str]:
|
||||
out = []
|
||||
for part in _SEGMENT_SPLIT.split(command or ""):
|
||||
seg = part.strip().lstrip("(").strip()
|
||||
while _ENV_ASSIGN.match(seg):
|
||||
seg = _ENV_ASSIGN.sub("", seg, count=1)
|
||||
seg = _squash(seg)
|
||||
if seg:
|
||||
out.append(seg)
|
||||
return out
|
||||
|
||||
|
||||
def _pairs(match: str) -> dict[str, str] | None:
|
||||
"""`a=b, c=d` → {"a": "b", "c": "d"}; None when it is not that shape."""
|
||||
out: dict[str, str] = {}
|
||||
for part in (match or "").split(","):
|
||||
key, sep, value = part.partition("=")
|
||||
if not sep or not key.strip():
|
||||
return None
|
||||
out[key.strip().lower()] = value.strip().lower()
|
||||
return out or None
|
||||
|
||||
|
||||
def action_matches(action: Action, tool: str, tool_input: dict | None) -> bool:
|
||||
if tool_key(action.tool) != tool_key(tool):
|
||||
return False
|
||||
if not action.match:
|
||||
return True
|
||||
tool_input = tool_input or {}
|
||||
command = tool_input.get(COMMAND_FIELD)
|
||||
if isinstance(command, str):
|
||||
want = _squash(action.match)
|
||||
return any(seg == want or seg.startswith(want + " ")
|
||||
for seg in _segments(command))
|
||||
pairs = _pairs(action.match)
|
||||
if pairs is None:
|
||||
return False
|
||||
given = {str(k).lower(): v for k, v in tool_input.items()}
|
||||
return all(
|
||||
str(given.get(k, "")).strip().lower() == v for k, v in pairs.items()
|
||||
)
|
||||
|
||||
|
||||
def effective_actions(mappings: Iterable) -> list[tuple[Action, str]]:
|
||||
"""The defaults this install has not removed, then its own additions.
|
||||
|
||||
`mappings` are MomentMapping rows, or anything with the same four fields.
|
||||
"""
|
||||
removed: set[tuple[str, str, str]] = set()
|
||||
added: list[Action] = []
|
||||
for m in mappings:
|
||||
key = (tool_key(m.tool), _squash(m.match), m.moment)
|
||||
if m.effect == REMOVE:
|
||||
removed.add(key)
|
||||
elif m.effect == ADD:
|
||||
added.append(Action(m.tool, _squash(m.match), m.moment))
|
||||
out = [
|
||||
(a, DEFAULT) for a in DEFAULT_ACTIONS
|
||||
if (tool_key(a.tool), a.match, a.moment) not in removed
|
||||
]
|
||||
out.extend((a, INSTALL) for a in added)
|
||||
return out
|
||||
|
||||
|
||||
def skill_name(tool: str, tool_input: dict | None) -> str:
|
||||
"""The procedure a skill-loader call loads, lowercased; "" for any other call."""
|
||||
if tool_key(tool) != SKILL_TOOL:
|
||||
return ""
|
||||
return str((tool_input or {}).get(SKILL_FIELD) or "").strip().lower()
|
||||
|
||||
|
||||
def declared_moments(data: dict | None) -> list[str]:
|
||||
"""The moments a stored process declares — the valid ones, in its order.
|
||||
|
||||
Read defensively: the field is written through require_moments, but a
|
||||
moment the catalog later drops must stop firing, not break the load.
|
||||
"""
|
||||
raw = (data or {}).get(PROCESS_MOMENTS_FIELD) or []
|
||||
if isinstance(raw, str):
|
||||
raw = [raw]
|
||||
return [m for m in dict.fromkeys(str(n).strip().lower() for n in raw)
|
||||
if catalog.is_moment(m)]
|
||||
|
||||
|
||||
def resolve(
|
||||
tool: str, tool_input: dict | None, mappings: Iterable = (),
|
||||
declared: Iterable[str] = (),
|
||||
) -> list[dict]:
|
||||
"""Every moment this call reaches, each with the action that reached it.
|
||||
|
||||
One entry per moment, in catalog order. When two actions reach the same
|
||||
moment the more specific one — the longer match — is the one named, since
|
||||
"reached by `git push`" says more than "reached by Bash".
|
||||
|
||||
`declared` are the moments the procedure this call loads says it is for
|
||||
(a stored process's own field); each is reached by the load itself.
|
||||
"""
|
||||
best: dict[str, dict] = {}
|
||||
for action, via in effective_actions(mappings):
|
||||
if not action_matches(action, tool, tool_input):
|
||||
continue
|
||||
held = best.get(action.moment)
|
||||
if held is None or len(action.match) > len(held["match"]):
|
||||
best[action.moment] = {
|
||||
"moment": action.moment, "tool": action.tool,
|
||||
"match": action.match, "via": via,
|
||||
}
|
||||
|
||||
name = skill_name(tool, tool_input)
|
||||
if name:
|
||||
moment = catalog.SKILL_PREFIX + name
|
||||
if catalog.is_moment(moment):
|
||||
best[moment] = {"moment": moment, "tool": tool,
|
||||
"match": f"{SKILL_FIELD}={name}", "via": DEFAULT}
|
||||
for moment in declared:
|
||||
best.setdefault(moment, {"moment": moment, "tool": tool,
|
||||
"match": f"{SKILL_FIELD}={name}", "via": PROCESS})
|
||||
|
||||
order = {name: i for i, name in enumerate(catalog.MOMENTS)}
|
||||
return sorted(best.values(), key=lambda h: (order.get(h["moment"], len(order)), h["moment"]))
|
||||
|
||||
|
||||
def _sample_input(tool: str, match: str) -> dict:
|
||||
"""A call that this action would describe — for reporting what it reaches."""
|
||||
if not match:
|
||||
return {}
|
||||
if tool_key(tool) in COMMAND_TOOLS:
|
||||
return {COMMAND_FIELD: match}
|
||||
return dict(_pairs(match) or {})
|
||||
|
||||
|
||||
def _clean(tool: str, match: str, moment: str) -> tuple[str, str, str]:
|
||||
"""Validate a mapping's three parts, or refuse with what would work."""
|
||||
bare = (tool or "").strip()
|
||||
if bare.startswith("mcp__"):
|
||||
bare = bare.rsplit("__", 1)[-1]
|
||||
if not bare:
|
||||
raise ValueError("tool is required — the tool as the harness names it, e.g. Bash or update_task")
|
||||
clean_match = _squash(match)
|
||||
if clean_match and tool_key(bare) not in COMMAND_TOOLS and _pairs(clean_match) is None:
|
||||
raise ValueError(
|
||||
f"{bare} does not run a command, so `match` names its arguments as "
|
||||
f"field=value pairs (e.g. status=done), not {clean_match!r}. Leave "
|
||||
f"it empty to map every {bare} call."
|
||||
)
|
||||
return bare, clean_match, catalog.require_moment(moment)
|
||||
|
||||
|
||||
def _is_default(tool: str, match: str, moment: str) -> bool:
|
||||
return any(
|
||||
tool_key(a.tool) == tool_key(tool) and a.match == match and a.moment == moment
|
||||
for a in DEFAULT_ACTIONS
|
||||
)
|
||||
|
||||
|
||||
# ── the install's own ─────────────────────────────────────────────────────
|
||||
|
||||
async def list_mappings(user_id: int) -> list[MomentMapping]:
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(MomentMapping)
|
||||
.where(MomentMapping.user_id == user_id)
|
||||
.order_by(MomentMapping.id)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def moments_for(user_id: int, tool: str, tool_input: dict | None) -> list[dict]:
|
||||
"""The moments a call reaches for this user.
|
||||
|
||||
Fails OPEN to the defaults: an unreadable mapping table must cost the
|
||||
install its corrections, not every moment.
|
||||
"""
|
||||
try:
|
||||
mappings = await list_mappings(user_id)
|
||||
except Exception:
|
||||
logger.warning("moment mappings unreadable; using the defaults", exc_info=True)
|
||||
mappings = []
|
||||
name = skill_name(tool, tool_input)
|
||||
declared = (await process_moments(user_id, name)
|
||||
if name.startswith(PROCESS_SKILL_PREFIX) else [])
|
||||
return resolve(tool, tool_input, mappings, declared)
|
||||
|
||||
|
||||
async def process_moments(user_id: int, skill: str) -> list[str]:
|
||||
"""The moments the stored process behind `scribe-proc-<slug>` declares.
|
||||
|
||||
The slug is resolved through the same manifest the sync script wrote the
|
||||
skill from, so the two agree on which process a slug names. Fails open to
|
||||
none: a process that cannot be read still loads, it just reaches no more
|
||||
than its own `skill.<name>`.
|
||||
"""
|
||||
from grimoire.services import notes as notes_svc
|
||||
from grimoire.services import plugin_context
|
||||
|
||||
slug = skill[len(PROCESS_SKILL_PREFIX):]
|
||||
try:
|
||||
manifest = await plugin_context.build_process_manifest(user_id)
|
||||
entry = next((p for p in manifest.get("processes", []) if p.get("slug") == slug), None)
|
||||
if entry is None:
|
||||
return []
|
||||
loaded = await notes_svc.get_note_for_user(user_id, int(entry["id"]))
|
||||
note = loaded[0] if loaded else None
|
||||
return declared_moments(note.data if note is not None else None)
|
||||
except Exception:
|
||||
logger.debug("process moments for %s unreadable", skill, exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
async def _find(session, user_id: int, tool: str, match: str, moment: str):
|
||||
rows = await session.execute(
|
||||
select(MomentMapping).where(
|
||||
MomentMapping.user_id == user_id,
|
||||
MomentMapping.moment == moment,
|
||||
MomentMapping.match == match,
|
||||
)
|
||||
)
|
||||
return next((r for r in rows.scalars().all() if tool_key(r.tool) == tool_key(tool)), None)
|
||||
|
||||
|
||||
async def _reaches(user_id: int, tool: str, match: str) -> list[dict]:
|
||||
return resolve(tool, _sample_input(tool, match), await list_mappings(user_id))
|
||||
|
||||
|
||||
async def map_action(
|
||||
user_id: int, tool: str, match: str, moment: str,
|
||||
*, reason: str = "", actor: str = "model",
|
||||
) -> dict:
|
||||
"""Make an action reach a moment for this install.
|
||||
|
||||
Mapping a shipped default this install had removed restores it; mapping
|
||||
one already in force changes nothing and says so.
|
||||
"""
|
||||
tool, match, moment = _clean(tool, match, moment)
|
||||
reason = (reason or "").strip()
|
||||
async with async_session() as session:
|
||||
row = await _find(session, user_id, tool, match, moment)
|
||||
if _is_default(tool, match, moment):
|
||||
if row is not None and row.effect == REMOVE:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
change = "restored the shipped default"
|
||||
else:
|
||||
change = "already a shipped default — nothing to add"
|
||||
elif row is not None:
|
||||
row.reason = reason or row.reason
|
||||
row.actor = actor
|
||||
await session.commit()
|
||||
change = "already mapped — reason updated" if reason else "already mapped"
|
||||
else:
|
||||
session.add(MomentMapping(
|
||||
user_id=user_id, tool=tool, match=match, moment=moment,
|
||||
effect=ADD, reason=reason, actor=actor,
|
||||
))
|
||||
await session.commit()
|
||||
change = "mapped"
|
||||
return {
|
||||
"change": change, "tool": tool, "match": match, "moment": moment,
|
||||
"now_reaches": await _reaches(user_id, tool, match),
|
||||
}
|
||||
|
||||
|
||||
async def unmap_action(
|
||||
user_id: int, tool: str, match: str, moment: str,
|
||||
*, reason: str = "", actor: str = "model",
|
||||
) -> dict:
|
||||
"""Stop an action reaching a moment for this install.
|
||||
|
||||
An install's own mapping is deleted. A shipped default is switched off by
|
||||
a stored removal, so the next release does not switch it back on.
|
||||
"""
|
||||
tool, match, moment = _clean(tool, match, moment)
|
||||
reason = (reason or "").strip()
|
||||
async with async_session() as session:
|
||||
row = await _find(session, user_id, tool, match, moment)
|
||||
if row is not None and row.effect == ADD:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
change = "removed this install's mapping"
|
||||
elif _is_default(tool, match, moment):
|
||||
if row is None:
|
||||
session.add(MomentMapping(
|
||||
user_id=user_id, tool=tool, match=match, moment=moment,
|
||||
effect=REMOVE, reason=reason, actor=actor,
|
||||
))
|
||||
await session.commit()
|
||||
change = "switched off the shipped default"
|
||||
else:
|
||||
change = "the shipped default was already off"
|
||||
else:
|
||||
raise ValueError(
|
||||
f"nothing maps {tool}{' ' + repr(match) if match else ''} onto "
|
||||
f"{moment}, so there is nothing to remove. list_moments shows "
|
||||
f"which actions reach each moment."
|
||||
)
|
||||
return {
|
||||
"change": change, "tool": tool, "match": match, "moment": moment,
|
||||
"now_reaches": await _reaches(user_id, tool, match),
|
||||
}
|
||||
|
||||
|
||||
async def actions_by_moment(user_id: int) -> dict:
|
||||
"""Each moment's actions in force, and the defaults this install removed."""
|
||||
mappings = await list_mappings(user_id)
|
||||
by_moment: dict[str, list[dict]] = {}
|
||||
for action, via in effective_actions(mappings):
|
||||
by_moment.setdefault(action.moment, []).append(
|
||||
{"tool": action.tool, "match": action.match, "via": via}
|
||||
)
|
||||
removed = [m.to_dict() for m in mappings if m.effect == REMOVE]
|
||||
return {"actions": by_moment, "removed_defaults": removed}
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Delivering the rules mounted on a moment, through every door.
|
||||
|
||||
One function decides what an act reaches and what arrives with it; the doors
|
||||
differ only in how they carry the answer:
|
||||
|
||||
- the plugin's catch-all PreToolUse hook → `/api/plugin/moment`, with the
|
||||
session ledger, so a rule already named this session is cited not repeated;
|
||||
- Grimoire's own MCP tools → `attach_moment_rules`, in the tool's response, so
|
||||
a client without the plugin still gets `work.finish` when a task closes;
|
||||
- the Stop hook → `deliver_moments`, for the reply moments no tool call marks.
|
||||
|
||||
The pipeline stage is `retrieval_pipeline.run_moment_arm`; this module only
|
||||
resolves the act to its moments and hands both doors the same result.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from grimoire.services import moment_actions, reply_shapes
|
||||
from grimoire.services import retrieval_pipeline as rp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _io() -> rp.MomentIO:
|
||||
"""Resolved at CALL time, so a test patching either module's name is seen."""
|
||||
from grimoire.services import rule_usage, rulebooks
|
||||
|
||||
return rp.MomentIO(
|
||||
lookup=rulebooks.rules_on_moments,
|
||||
record_rule_surfaced=rule_usage.record_rule_surfaced,
|
||||
)
|
||||
|
||||
|
||||
async def reachable_tools(user_id: int) -> list[str]:
|
||||
"""The tools a call to which could deliver a mounted rule, as tool keys.
|
||||
|
||||
What the plugin's catch-all hook reads once per session window, so the
|
||||
calls that cannot reach a mounted rule — most of them, and every one on
|
||||
an install that has mounted nothing — never leave the machine. An action
|
||||
counts when its moment carries a mount or a default reply shape. The
|
||||
skill loader counts whenever anything is mounted: a stored process
|
||||
declares its own moments, which only the load itself can resolve, and a
|
||||
load is rare enough that asking costs nothing.
|
||||
"""
|
||||
from grimoire.services import rulebooks
|
||||
|
||||
# A moment that carries a default reply shape is worth a
|
||||
# request whether or not anything is mounted on it: the shape is product,
|
||||
# so every install has it.
|
||||
shaped = {s.moment for s in reply_shapes.SHAPES.values() if s.key != reply_shapes.CORE_KEY}
|
||||
mounted = set(await rulebooks.mounted_moments(user_id))
|
||||
wanted = mounted | shaped
|
||||
mappings = await moment_actions.list_mappings(user_id)
|
||||
keys = {
|
||||
moment_actions.tool_key(action.tool)
|
||||
for action, _via in moment_actions.effective_actions(mappings)
|
||||
if action.moment in wanted
|
||||
}
|
||||
if mounted:
|
||||
keys.add(moment_actions.SKILL_TOOL)
|
||||
return sorted(keys)
|
||||
|
||||
|
||||
async def deliver_moments(
|
||||
user_id: int, reached: list[dict], *, project_id: int | None = None,
|
||||
exclude: frozenset[int] = frozenset(), held: frozenset[int] = frozenset(),
|
||||
) -> rp.RuleResult:
|
||||
"""The mounted rules for moments already known to have happened."""
|
||||
return await rp.run_moment_arm(
|
||||
reached,
|
||||
rp.RuleMoment(user_id=user_id, query="", project_id=project_id,
|
||||
exclude=exclude, held=held),
|
||||
io=_io(),
|
||||
)
|
||||
|
||||
|
||||
async def deliver_for_act(
|
||||
user_id: int, tool: str, tool_input: dict | None, *,
|
||||
project_id: int | None = None,
|
||||
exclude: frozenset[int] = frozenset(), held: frozenset[int] = frozenset(),
|
||||
) -> tuple[list[dict], rp.RuleResult]:
|
||||
"""Which moments this act reached, and the mounted rules they deliver."""
|
||||
reached = await moment_actions.moments_for(user_id, tool, tool_input or {})
|
||||
if not reached:
|
||||
return [], rp.RuleResult()
|
||||
result = await deliver_moments(
|
||||
user_id, reached, project_id=project_id, exclude=exclude, held=held,
|
||||
)
|
||||
return reached, result
|
||||
|
||||
|
||||
def shapes_for_act(reached: list[dict], seen: frozenset[str] = frozenset()) -> tuple[list[str], dict[str, str]]:
|
||||
"""The default reply shapes riding the moments this act reached.
|
||||
|
||||
Closing a task comes just before a completion report, a structured
|
||||
question is an ask, opening a plan comes before a plan is put up for
|
||||
review — so the shape for that reply arrives at the act, before the reply
|
||||
is written. In full the first time a session meets it, as its reminder
|
||||
after that; a door with no ledger passes `seen` empty.
|
||||
"""
|
||||
return reply_shapes.deliver(
|
||||
reply_shapes.for_moments([hit["moment"] for hit in reached]), seen,
|
||||
)
|
||||
|
||||
|
||||
# What the per-turn delivery says reached `reply.report`: the turn has not
|
||||
# ended yet, but every turn ends in a reply, and this is the last point
|
||||
# before it is written.
|
||||
_REACHED_BY_TURN = "the reply this turn will end with"
|
||||
|
||||
|
||||
async def deliver_for_turn(
|
||||
user_id: int, *, seen: frozenset[str] = frozenset(), project_id: int | None = None,
|
||||
exclude: frozenset[int] = frozenset(), held: frozenset[int] = frozenset(),
|
||||
) -> dict:
|
||||
"""What every turn carries before its reply is written.
|
||||
|
||||
The core reply shape — in full once per session and again after a
|
||||
compaction (the ledger is swept then), otherwise its one-line reminder —
|
||||
and the rules and preferences mounted on `reply.report`, under the same
|
||||
rule ledger as every other arm. The Stop hook's reply moment fires after
|
||||
the reply exists, which is too late to shape it and is kept as the
|
||||
backstop; this is the point before.
|
||||
|
||||
Returns `{context, rule_ids, shape_forms}`. Fails open to the core alone,
|
||||
and the core itself never fails: it is a constant.
|
||||
"""
|
||||
blocks, forms = reply_shapes.deliver([reply_shapes.core()], seen)
|
||||
rule_ids: list[int] = []
|
||||
try:
|
||||
reached = [{"moment": "reply.report", "tool": "", "match": _REACHED_BY_TURN,
|
||||
"via": moment_actions.DEFAULT}]
|
||||
result = await deliver_moments(
|
||||
user_id, reached, project_id=project_id, exclude=exclude, held=held,
|
||||
)
|
||||
blocks.extend(result.lines)
|
||||
rule_ids = result.rule_ids
|
||||
except Exception: # noqa: BLE001 - the mounted half never costs the core
|
||||
logger.debug("reply.report mounts not delivered for the turn", exc_info=True)
|
||||
await reply_shapes.record_delivery(user_id, forms, via="turn")
|
||||
return {"context": "\n\n".join(blocks), "rule_ids": rule_ids, "shape_forms": forms}
|
||||
|
||||
|
||||
async def attach_moment_rules(
|
||||
user_id: int, tool: str, arguments: dict | None, data: dict,
|
||||
) -> dict:
|
||||
"""Carry the rules mounted on this tool call's moments in its own response.
|
||||
|
||||
The MCP door: an in-band decoration on a payload
|
||||
already being returned, fail-open, and absent when there is nothing to
|
||||
say. The project is the record's own when it has one, else the caller's
|
||||
argument — a project's mounted rules belong to work in that project.
|
||||
|
||||
No session ledger reaches this door, so a mounted rule arrives every time
|
||||
its moment happens here. That is the right failure for the moments these
|
||||
tools mark: closing a task is exactly when a rule about finishing applies,
|
||||
however many tasks were closed before it.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
try:
|
||||
project_id = data.get("project_id") or (arguments or {}).get("project_id") or None
|
||||
_reached, result = await deliver_for_act(
|
||||
user_id, tool, arguments or {}, project_id=project_id,
|
||||
)
|
||||
if result.lines:
|
||||
data["moment_rules"] = {
|
||||
"lines": result.lines,
|
||||
"rule_ids": result.shown_rule_ids,
|
||||
"open_with": "get_rule(id)",
|
||||
}
|
||||
# The reply shape for this moment, in full: no ledger reaches this
|
||||
# door, and an act like closing a task is rare enough that the shape
|
||||
# arriving each time costs less than one report written without it.
|
||||
blocks, forms = shapes_for_act(_reached)
|
||||
if blocks:
|
||||
data["reply_shape"] = "\n\n".join(blocks)
|
||||
await reply_shapes.record_delivery(user_id, forms, via="mcp")
|
||||
except Exception: # noqa: BLE001 - a decoration never breaks the payload
|
||||
logger.debug("moment rules for %s could not be attached", tool, exc_info=True)
|
||||
return data
|
||||
|
||||
|
||||
# ── The reply moment: the backstop at the end of a turn ─────────────────
|
||||
#
|
||||
# The Stop hook's door. A reply is the
|
||||
# one act no tool call marks, and the moment where "please verify this on
|
||||
# your end" gets said — so it is checked twice over: the rules MOUNTED on the
|
||||
# reply moments (deterministic, somebody said they belong here), and the
|
||||
# reply text against every rule's trigger (the backstop for whatever the
|
||||
# earlier arms missed). An unopened rule from either half HOLDS the reply for
|
||||
# one read, in these words; the hook never holds the rewrite.
|
||||
|
||||
# The reply's head and tail. The embedder reads ~512 tokens, and the part of
|
||||
# a report that asks something of the reader — "let me know if it works" —
|
||||
# is at the END, where a head-only query would cut it off.
|
||||
_REPLY_HEAD_CHARS = 600
|
||||
_REPLY_TAIL_CHARS = 1200
|
||||
|
||||
_REACHED_BY_REPLY = "the reply that ends this turn"
|
||||
|
||||
|
||||
def reply_moments(reply: str) -> list[dict]:
|
||||
"""The moments a finished reply reaches: always `reply.report`, and
|
||||
`reply.ask` when a line of its prose ends in a question."""
|
||||
reached = [{"moment": "reply.report", "tool": "", "match": _REACHED_BY_REPLY,
|
||||
"via": moment_actions.DEFAULT}]
|
||||
in_code = False
|
||||
for line in (reply or "").splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_code = not in_code
|
||||
continue
|
||||
if not in_code and stripped.endswith("?"):
|
||||
reached.append({"moment": "reply.ask", "tool": "", "match": "a question in the reply",
|
||||
"via": moment_actions.DEFAULT})
|
||||
break
|
||||
return reached
|
||||
|
||||
|
||||
def _reply_query(reply: str) -> str:
|
||||
text = " ".join((reply or "").split())
|
||||
if len(text) <= _REPLY_HEAD_CHARS + _REPLY_TAIL_CHARS:
|
||||
return text
|
||||
return text[:_REPLY_HEAD_CHARS] + " … " + text[-_REPLY_TAIL_CHARS:]
|
||||
|
||||
|
||||
def reply_hold_reason(held: list[dict]) -> str:
|
||||
"""The text the agent reads INSTEAD of its reply going out.
|
||||
|
||||
A practice, not a prohibition, on the act checkpoint's model:
|
||||
nothing here knows the reply is wrong. It names each rule with why it was
|
||||
raised, gives the remedy as one call per rule, and says the reply may go
|
||||
out unchanged — so a reader who finds the rules beside the point is out in
|
||||
as many calls as there are rules, and is never held twice.
|
||||
"""
|
||||
if not held:
|
||||
return ""
|
||||
parts = []
|
||||
for item in held:
|
||||
why = (f"mounted on {item['moment']}" if item.get("moment")
|
||||
else f"scores {item['score']} against this reply")
|
||||
trigger = f"; it applies when {item['trigger']}" if item.get("trigger") else ""
|
||||
parts.append(f"“{item['title']}” ({why}{trigger}) — get_rule({item['rule_id']})")
|
||||
noun = "a standing rule" if len(held) == 1 else "standing rules"
|
||||
# A mount that does not bear on this reply is a misfire worth one call:
|
||||
# reports gather into an unmount proposal for the operator.
|
||||
mounted = [item for item in held if item.get("moment")]
|
||||
misfire = (
|
||||
" If one mounted on a moment has nothing to do with this reply, "
|
||||
f"rule_misfired({mounted[0]['rule_id']}, \"{mounted[0]['moment']}\", why) "
|
||||
"records that."
|
||||
if mounted else ""
|
||||
)
|
||||
return (
|
||||
f"Held for one read before this reply goes out: {noun} this session "
|
||||
f"has not opened: " + "; ".join(parts) + ". Read "
|
||||
+ ("it" if len(held) == 1 else "them")
|
||||
+ ", then send the reply — unchanged if it already does what the rule "
|
||||
"asks, which is a judgement only you can make." + misfire
|
||||
+ " This check runs once per rule; the rewrite is never held."
|
||||
)
|
||||
|
||||
|
||||
async def reply_hold(
|
||||
user_id: int, reply: str, *, project_id: int | None = None,
|
||||
exclude: frozenset[int] = frozenset(), held: frozenset[int] = frozenset(),
|
||||
stopped: frozenset[int] = frozenset(),
|
||||
) -> dict:
|
||||
"""Whether this finished reply is held, and in what words.
|
||||
|
||||
`held` is what the session OPENED (the act checkpoint's exemption);
|
||||
`stopped` is what an earlier hold already put in front of it, so a rule
|
||||
holds a session once. Past the act checkpoint's per-session cap nothing
|
||||
holds — a mis-set bar degrades to a quiet session, never a stuck one.
|
||||
|
||||
Returns {} or {"reason", "rule_ids", "moments"}. Fails open to {}.
|
||||
"""
|
||||
from grimoire.services import plugin_context as pc
|
||||
from grimoire.services import rule_usage, rulebooks
|
||||
from grimoire.services.retrieval_surfaces import budget_for, floor_for
|
||||
|
||||
try:
|
||||
if not (reply or "").strip() or len(stopped) >= pc.CHECKPOINT_SESSION_CAP:
|
||||
return {}
|
||||
reached = reply_moments(reply)
|
||||
names = [hit["moment"] for hit in reached]
|
||||
skip = set(held) | set(stopped)
|
||||
room = pc.CHECKPOINT_SESSION_CAP - len(stopped)
|
||||
out: list[dict] = []
|
||||
|
||||
# The mounted half: deterministic, so every unopened RULE on these
|
||||
# moments holds. A preference claims no such force.
|
||||
for rule, at in await rulebooks.rules_on_moments(user_id, names, project_id or None):
|
||||
if rule.id in skip or rule.kind == "preference":
|
||||
continue
|
||||
out.append({"rule_id": rule.id, "title": rule.title, "moment": at,
|
||||
"trigger": (rule.when_to_apply or "").strip()})
|
||||
# Trimmed BEFORE recording: a rule the cap cut was shown to nobody.
|
||||
out = out[:room]
|
||||
mounted = [item["rule_id"] for item in out]
|
||||
fresh = [rid for rid in mounted if rid not in exclude]
|
||||
if fresh:
|
||||
try:
|
||||
rule_usage.record_rule_surfaced(
|
||||
user_id=user_id, rule_ids=fresh, source=rp.MOMENT_RULE_SOURCE,
|
||||
detail={item["rule_id"]: item["moment"] for item in out
|
||||
if item["rule_id"] in fresh},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - observation never breaks the observed
|
||||
logger.debug("reply mount surfacing not recorded", exc_info=True)
|
||||
|
||||
# The semantic half: the backstop. A mounted rule already holding is
|
||||
# treated as held here, so one rule is never named twice.
|
||||
floor = await floor_for(user_id, rp.REPLY_RULE.source)
|
||||
result = await rp.run_rule_arm(
|
||||
rp.REPLY_RULE,
|
||||
rp.RuleMoment(
|
||||
user_id=user_id, query=_reply_query(reply), project_id=project_id,
|
||||
where="to this reply", checkpoint_where="this reply",
|
||||
exclude=exclude, held=frozenset(skip | set(mounted)),
|
||||
),
|
||||
floor=floor, budget=await budget_for(user_id, rp.REPLY_RULE.source),
|
||||
io=pc._rule_io(), checkpoint_floor=floor,
|
||||
)
|
||||
if result.checkpoint and len(out) < room:
|
||||
cp = result.checkpoint
|
||||
out.append({"rule_id": cp["rule_id"], "title": cp["title"],
|
||||
"score": cp["score"], "trigger": cp.get("trigger", "")})
|
||||
|
||||
if not out:
|
||||
return {}
|
||||
return {
|
||||
"reason": reply_hold_reason(out),
|
||||
"rule_ids": [item["rule_id"] for item in out],
|
||||
"moments": names,
|
||||
}
|
||||
except Exception: # noqa: BLE001 - a recall aid never stops a session by failing
|
||||
logger.debug("reply hold failed", exc_info=True)
|
||||
return {}
|
||||
@@ -0,0 +1,171 @@
|
||||
"""The moments of work that rules mount on.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
Semantic retrieval answers "what does this text resemble?", and most rules are
|
||||
not ABOUT a topic — they belong to a MOMENT in the work. A rule about when work
|
||||
counts as finished governs the moment work is sent out, checked, closed and
|
||||
reported, and nothing said at those moments needs to resemble the rule. Matched
|
||||
on topic, it scores below every bar; mounted on its moments, it cannot miss.
|
||||
|
||||
So a rule names the moments it belongs to, and when one happens every rule
|
||||
mounted on it arrives — by lookup, with no embedding and no score. Semantic
|
||||
retrieval stays the second net, for whatever nobody mounted.
|
||||
|
||||
WHY THE CATALOG IS CODE, NOT DATA
|
||||
|
||||
The moments are the product's vocabulary, not an install's. Every install needs
|
||||
the same mount points, or a rule written on one could not mean the same thing on
|
||||
another, and the plugin's hooks, the skills that declare moments and the
|
||||
instruction surfaces all name them. What varies per install is which ACTIONS
|
||||
reach a moment — one operator delivers with a push, another with a deploy
|
||||
script, a third by sending a document — and that mapping is data (`moment_actions`). The
|
||||
moment is generic; the action that reaches it is local.
|
||||
|
||||
HOW TO WRITE ONE
|
||||
|
||||
Domain-neutral: Grimoire's work may be software, configuration,
|
||||
infrastructure or anything else a person drives through an agent, so a
|
||||
definition names the moment in words every one of those recognises. The test
|
||||
beside this module refuses software-only vocabulary in both fields. Concrete
|
||||
actions belong in the default mappings, where they are examples of reaching a
|
||||
moment rather than the meaning of it.
|
||||
|
||||
Names are expensive to change once rules point at them. Adding one is cheap;
|
||||
renaming one is a migration of every mount.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Moment:
|
||||
"""One point in the work that rules can mount on."""
|
||||
|
||||
name: str
|
||||
"""`<area>.<verb>` — the string a mount stores and a hook reports."""
|
||||
|
||||
means: str
|
||||
"""What is happening at this moment, in one line, for any kind of work."""
|
||||
|
||||
reached_by: str
|
||||
"""The kinds of action that typically reach it, said generically."""
|
||||
|
||||
|
||||
def _m(name: str, means: str, reached_by: str) -> tuple[str, Moment]:
|
||||
return name, Moment(name=name, means=means, reached_by=reached_by)
|
||||
|
||||
|
||||
MOMENTS: dict[str, Moment] = dict([
|
||||
_m("session.start",
|
||||
"a working session begins, or resumes after its context was summarized",
|
||||
"opening a session; resuming after the earlier conversation was summarized"),
|
||||
_m("work.start",
|
||||
"taking up a piece of work",
|
||||
"marking a task as in progress; beginning a step of a plan"),
|
||||
_m("work.plan",
|
||||
"designing an approach before carrying it out",
|
||||
"opening a plan; entering a planning mode; loading a planning procedure"),
|
||||
_m("work.change",
|
||||
"altering the thing being worked on",
|
||||
"editing or creating a file; changing a setting or a document"),
|
||||
_m("work.run",
|
||||
"carrying out an action in the environment",
|
||||
"running a command or a script"),
|
||||
_m("work.verify",
|
||||
"checking that the work does what it should",
|
||||
"running a check; reading the result of an automated check; "
|
||||
"loading a verification procedure"),
|
||||
_m("work.deliver",
|
||||
"sending work beyond the place it was made",
|
||||
"publishing, merging, releasing, deploying or sending it to someone"),
|
||||
_m("work.finish",
|
||||
"declaring a piece of work done, or abandoning it",
|
||||
"closing a task or an issue; marking it done or cancelled"),
|
||||
_m("work.record",
|
||||
"writing down what was learned or decided",
|
||||
"creating a note, a lesson, a rule or a reusable example"),
|
||||
_m("work.debug",
|
||||
"working out why something does not behave as expected",
|
||||
"reading a failure; loading a diagnosis procedure"),
|
||||
_m("work.delegate",
|
||||
"handing part of the work to another agent",
|
||||
"dispatching a helper agent"),
|
||||
_m("env.reach",
|
||||
"touching a machine or a service outside the workspace",
|
||||
"connecting to a host; calling a remote service; inspecting what runs there"),
|
||||
_m("reply.report",
|
||||
"reporting results to the person the work is for",
|
||||
"the reply that ends a turn; loading a reporting procedure"),
|
||||
_m("reply.ask",
|
||||
"asking the person the work is for to decide or to act",
|
||||
"a question in the reply; a structured question to the person"),
|
||||
])
|
||||
|
||||
# A FAMILY rather than entries: one moment per named procedure, reached when it
|
||||
# is loaded. The names are the procedures' own (bundled skills, stored
|
||||
# processes), so they cannot be listed here — only their shape can.
|
||||
SKILL_PREFIX = "skill."
|
||||
SKILL_FAMILY = Moment(
|
||||
name=f"{SKILL_PREFIX}<name>",
|
||||
means="a named procedure is loaded",
|
||||
reached_by="loading a skill or a stored process by its name",
|
||||
)
|
||||
_SKILL_NAME = re.compile(r"^[a-z0-9][a-z0-9_:-]*$")
|
||||
|
||||
|
||||
def is_moment(name: str) -> bool:
|
||||
"""Is this a name a rule can mount on — a catalog moment, or skill.<name>?"""
|
||||
if name in MOMENTS:
|
||||
return True
|
||||
if name.startswith(SKILL_PREFIX):
|
||||
return bool(_SKILL_NAME.match(name[len(SKILL_PREFIX):]))
|
||||
return False
|
||||
|
||||
|
||||
def require_moment(name: str) -> str:
|
||||
"""The name, normalised, or a refusal that lists what is available.
|
||||
|
||||
A typo'd moment must not be mountable: the mount would be stored, read
|
||||
back, and never fire — a rule that looks attached and is not, which is the
|
||||
silent failure moments exist to end.
|
||||
"""
|
||||
clean = (name or "").strip().lower()
|
||||
if is_moment(clean):
|
||||
return clean
|
||||
raise ValueError(
|
||||
f"unknown moment {name!r}. Moments are: "
|
||||
+ ", ".join(MOMENTS)
|
||||
+ f", or {SKILL_FAMILY.name} for a named procedure"
|
||||
)
|
||||
|
||||
|
||||
def require_moments(names: list[str] | None) -> list[str] | None:
|
||||
"""A rule's moments, normalised and de-duplicated — or the first refusal.
|
||||
|
||||
None passes through as None, which every rule door reads as "leave the
|
||||
mounts alone"; a list, [] included, is the whole set after the write.
|
||||
All are checked before any is stored, so a refusal leaves nothing
|
||||
half-mounted.
|
||||
"""
|
||||
if names is None:
|
||||
return None
|
||||
if isinstance(names, str):
|
||||
names = [names]
|
||||
return list(dict.fromkeys(require_moment(n) for n in names))
|
||||
|
||||
def catalog() -> dict:
|
||||
"""The catalog as plain data, for the MCP tool and the REST door alike."""
|
||||
return {
|
||||
"moments": [
|
||||
{"name": m.name, "means": m.means, "reached_by": m.reached_by}
|
||||
for m in MOMENTS.values()
|
||||
],
|
||||
"families": [
|
||||
{"name": SKILL_FAMILY.name, "prefix": SKILL_PREFIX,
|
||||
"means": SKILL_FAMILY.means, "reached_by": SKILL_FAMILY.reached_by},
|
||||
],
|
||||
"total": len(MOMENTS),
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note_draft import NoteDraft
|
||||
|
||||
|
||||
async def upsert_draft(
|
||||
user_id: int,
|
||||
note_id: int,
|
||||
proposed_body: str,
|
||||
original_body: str,
|
||||
instruction: str,
|
||||
scope: str,
|
||||
) -> NoteDraft:
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
text("""
|
||||
INSERT INTO note_drafts (note_id, user_id, proposed_body, original_body, instruction, scope)
|
||||
VALUES (:note_id, :user_id, :proposed_body, :original_body, :instruction, :scope)
|
||||
ON CONFLICT (note_id, user_id) DO UPDATE SET
|
||||
proposed_body = EXCLUDED.proposed_body,
|
||||
original_body = EXCLUDED.original_body,
|
||||
instruction = EXCLUDED.instruction,
|
||||
scope = EXCLUDED.scope,
|
||||
updated_at = NOW()
|
||||
""").bindparams(
|
||||
note_id=note_id,
|
||||
user_id=user_id,
|
||||
proposed_body=proposed_body,
|
||||
original_body=original_body,
|
||||
instruction=instruction,
|
||||
scope=scope,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
result = await session.execute(
|
||||
select(NoteDraft).where(
|
||||
NoteDraft.note_id == note_id, NoteDraft.user_id == user_id
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def get_draft(user_id: int, note_id: int) -> NoteDraft | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(NoteDraft).where(
|
||||
NoteDraft.note_id == note_id, NoteDraft.user_id == user_id
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def delete_draft(user_id: int, note_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(NoteDraft).where(
|
||||
NoteDraft.note_id == note_id, NoteDraft.user_id == user_id
|
||||
)
|
||||
)
|
||||
draft = result.scalars().first()
|
||||
if draft is None:
|
||||
return False
|
||||
await session.delete(draft)
|
||||
await session.commit()
|
||||
return True
|
||||
@@ -0,0 +1,370 @@
|
||||
"""Note usage telemetry — was a surfaced note ever actually pulled?
|
||||
|
||||
Two event streams, deliberately independent:
|
||||
|
||||
- SURFACED: we put this note's title in front of the agent (auto-inject, or
|
||||
either arm of the write-path prior-art trigger).
|
||||
- PULLED: someone then opened it in full (get_snippet / get_note / the REST
|
||||
detail route).
|
||||
|
||||
The ratio between them is the signal. A snippet surfaced forty times and never
|
||||
pulled is not neutral — it occupies the injection budget on every future turn
|
||||
and dilutes the menu — so this is what makes dead weight visible and prunable.
|
||||
|
||||
Design notes (mirrors retrieval_telemetry, for the same reasons):
|
||||
- Writes are fire-and-forget. `record_surfaced` / `record_pulled` extract
|
||||
plain ints synchronously and schedule the insert as a background task, so
|
||||
telemetry never adds latency to — or can break — the surface it observes.
|
||||
- Failures degrade, but they must not degrade SILENTLY. Swallowed into a
|
||||
debug log, a failure leaves every counter at zero while surfacing fires —
|
||||
a total outage indistinguishable from "nobody uses this". A subsystem
|
||||
whose every failure mode is invisible cannot report its own death, so
|
||||
failures log at WARNING and drop one AppLog error row per process per
|
||||
site, where the admin UI shows it.
|
||||
- Reads (`usage_for_notes`) are NOT fire-and-forget — a readout the caller
|
||||
awaits, aggregated in one round-trip for a whole page of snippets rather
|
||||
than per row.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.note_usage import PULLED, SURFACED, NoteUsageEvent
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.services.background import report_telemetry_failure
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Strong references to in-flight inserts. The event loop keeps only a WEAK
|
||||
# reference to a task, so a fire-and-forget create_task with no other holder
|
||||
# can be garbage-collected before it completes — a write that never errors and
|
||||
# never lands. The done-callback discard keeps the set from growing.
|
||||
_pending: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
async def _report_failure(site: str) -> None:
|
||||
"""This subsystem's canary, now the shared one.
|
||||
|
||||
The per-site dedup, the WARNING and the single AppLog row all moved to
|
||||
`background.report_telemetry_failure` unchanged when `rule_usage` needed
|
||||
the identical behaviour — two hand-kept copies of a thing whose whole job
|
||||
is to be reliable is the wrong number. `retrieval_telemetry` deliberately
|
||||
still has its own: its canary is a different shape (one process-wide flag,
|
||||
no AppLog row), so repointing it would change behaviour rather than
|
||||
consolidate it.
|
||||
"""
|
||||
await report_telemetry_failure("note_usage", site)
|
||||
|
||||
|
||||
async def _insert_events(rows: list[dict]) -> None:
|
||||
"""Persist usage rows. Best-effort: failures degrade, visibly."""
|
||||
try:
|
||||
async with async_session() as session:
|
||||
session.add_all([NoteUsageEvent(**row) for row in rows])
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await _report_failure("write")
|
||||
|
||||
|
||||
def _schedule(rows: list[dict]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(_insert_events(rows))
|
||||
except RuntimeError:
|
||||
# No running loop (sync context outside the app) — skip rather than
|
||||
# block. Every app path runs on the loop.
|
||||
logger.debug("note usage telemetry skipped — no running event loop")
|
||||
return
|
||||
_pending.add(task)
|
||||
task.add_done_callback(_pending.discard)
|
||||
|
||||
|
||||
def _project_or_none(project_id: int | None) -> int | None:
|
||||
"""0 and None both mean "no project reported" — store one of them.
|
||||
|
||||
Callers reach this from two conventions at once: the MCP tools spell "no
|
||||
project" as `0` (it is an int parameter with an int default), while the
|
||||
column is nullable. Folding them here keeps every call site from having to
|
||||
remember which one this function wants, and stops a row claiming it was
|
||||
read on project #0.
|
||||
"""
|
||||
try:
|
||||
pid = int(project_id or 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return pid or None
|
||||
|
||||
|
||||
def record_surfaced(
|
||||
*,
|
||||
user_id: int | None,
|
||||
note_ids: list[int] | set[int],
|
||||
source: str,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: record that these notes were shown to the agent.
|
||||
|
||||
Takes the whole menu at once — one insert per surfacing event, not per note
|
||||
— because a menu is a single decision and its rows should land together.
|
||||
|
||||
`project_id` is where the READER was, not where the record lives. Every
|
||||
surfacing arm knows it — it is the scope it just searched — so pass it;
|
||||
it is what makes "surfaced away from home" answerable without joining
|
||||
RetrievalLog. 0 is normalised to None: a project id of zero means "no
|
||||
project" everywhere else in this codebase, and storing it would read as
|
||||
project #0.
|
||||
"""
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"user_id": user_id,
|
||||
"note_id": int(nid),
|
||||
"event": SURFACED,
|
||||
"source": source,
|
||||
"project_id": _project_or_none(project_id),
|
||||
}
|
||||
for nid in note_ids
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("note usage payload build failed", exc_info=True)
|
||||
return
|
||||
_schedule(rows)
|
||||
|
||||
|
||||
def record_pulled(
|
||||
*,
|
||||
user_id: int | None,
|
||||
note_id: int,
|
||||
source: str,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: record that a note was opened in full.
|
||||
|
||||
`project_id` is where the READER was — the caller's active project, never
|
||||
the record's own. Compared against the record's `project_id`, it answers
|
||||
whether this was opened somewhere other than where it was written, which
|
||||
is the evidence that a record TRANSFERRED.
|
||||
|
||||
Unlike a surfacing arm, a getter only knows what it was handed, so this
|
||||
stays optional and null is an ordinary answer meaning "not reported". A
|
||||
pull with no project is still a pull: it counts toward dead-weight
|
||||
detection and simply cannot speak to transfer.
|
||||
"""
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"user_id": user_id,
|
||||
"note_id": int(note_id),
|
||||
"event": PULLED,
|
||||
"source": source,
|
||||
"project_id": _project_or_none(project_id),
|
||||
}
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("note usage payload build failed", exc_info=True)
|
||||
return
|
||||
_schedule(rows)
|
||||
|
||||
|
||||
# Surfacings that are NOT a ranked choice. enter_project returns whatever the
|
||||
# top-N-by-recency happen to be; the skill sync installs every Process the
|
||||
# operator can reach. Counting those alongside auto-inject would make a note's
|
||||
# surfaced_count dominated by "it was recently updated in a project you
|
||||
# opened", and dead-weight detection would read that as popularity.
|
||||
# They still matter — a pull that follows one must not float unattributed — so
|
||||
# they land in their own bucket rather than not landing at all.
|
||||
AMBIENT_SOURCES = ("enter_project", "process_skill_sync")
|
||||
|
||||
|
||||
def empty_usage() -> dict:
|
||||
"""The zero readout — what a note with no recorded events looks like.
|
||||
|
||||
Callers render this shape unconditionally, so a note predating the table
|
||||
reads as "never surfaced, never pulled" rather than as a missing key.
|
||||
|
||||
`surfaced_count` is RANKED surfacings only — a scored surface chose this
|
||||
record. `ambient_count` is the rest (see AMBIENT_SOURCES). The split exists
|
||||
because the "high surfaced, zero pulls → dead weight" reading is only valid over surfacings that were choices.
|
||||
|
||||
`surfaced_away_count` / `pulled_away_count` are the subsets that happened
|
||||
on a project other than the one the record was written on — the evidence
|
||||
that a record TRANSFERRED, which is the claim the lesson kind rests on.
|
||||
Counted only where both projects are known: an
|
||||
event with no reader project, or a record with no project of its own,
|
||||
cannot speak to "away" and is left out rather than guessed.
|
||||
"""
|
||||
return {
|
||||
"surfaced_count": 0,
|
||||
"ambient_count": 0,
|
||||
"pull_count": 0,
|
||||
"last_surfaced_at": None,
|
||||
"last_pulled_at": None,
|
||||
"surfaced_away_count": 0,
|
||||
"pulled_away_count": 0,
|
||||
}
|
||||
|
||||
|
||||
async def usage_for_notes(note_ids: list[int]) -> dict[int, dict]:
|
||||
"""Aggregate usage for a set of notes: {note_id: {counts + timestamps}}.
|
||||
|
||||
One GROUP BY for the whole page rather than a query per row — this feeds a
|
||||
list view, so the per-row shape would be N+1 by construction. Notes with no
|
||||
events are returned with `empty_usage()` so the caller never has to
|
||||
distinguish "no events" from "not in the result".
|
||||
"""
|
||||
ids = [int(n) for n in note_ids]
|
||||
out: dict[int, dict] = {nid: empty_usage() for nid in ids}
|
||||
if not ids:
|
||||
return out
|
||||
|
||||
# Classified in SQL so the group count stays small: per note we get at most
|
||||
# (surfaced-ranked, surfaced-ambient, pulled) rather than one row per
|
||||
# distinct source. ONE labelled expression, grouped by its label — a second
|
||||
# case() instance in GROUP BY renders with its own expanding-IN bind names
|
||||
# under asyncpg, so the database sees two DIFFERENT expressions and rejects
|
||||
# the query with a GroupingError — and a swallowed rejection reads as
|
||||
# every counter at zero while the writes land fine.
|
||||
ambient = case(
|
||||
(NoteUsageEvent.source.in_(AMBIENT_SOURCES), True),
|
||||
else_=False,
|
||||
).label("ambient")
|
||||
try:
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
NoteUsageEvent.note_id,
|
||||
NoteUsageEvent.event,
|
||||
func.count().label("n"),
|
||||
func.max(NoteUsageEvent.created_at).label("last_at"),
|
||||
ambient,
|
||||
)
|
||||
.where(NoteUsageEvent.note_id.in_(ids))
|
||||
.group_by(
|
||||
NoteUsageEvent.note_id,
|
||||
NoteUsageEvent.event,
|
||||
ambient,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
# Away from home: the reader's project against the record's own.
|
||||
# A second aggregate rather than a column on the first, because
|
||||
# it needs the join to `notes` and the first must keep counting
|
||||
# events whose note has since been deleted (the table is FK-free
|
||||
# so that evidence outlives the row). Ranked surfacings only, for
|
||||
# the same reason surfaced_count is.
|
||||
away_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
NoteUsageEvent.note_id,
|
||||
NoteUsageEvent.event,
|
||||
func.count().label("n"),
|
||||
)
|
||||
.join(Note, Note.id == NoteUsageEvent.note_id)
|
||||
.where(
|
||||
NoteUsageEvent.note_id.in_(ids),
|
||||
NoteUsageEvent.project_id.is_not(None),
|
||||
Note.project_id.is_not(None),
|
||||
NoteUsageEvent.project_id != Note.project_id,
|
||||
NoteUsageEvent.source.not_in(AMBIENT_SOURCES),
|
||||
)
|
||||
.group_by(NoteUsageEvent.note_id, NoteUsageEvent.event)
|
||||
)
|
||||
).all()
|
||||
except Exception:
|
||||
# A telemetry readout must not be able to break the list it decorates —
|
||||
# but it must say it failed, or a broken readout is indistinguishable
|
||||
# from a corpus nobody uses.
|
||||
await _report_failure("readout")
|
||||
return out
|
||||
|
||||
for note_id, event, n, last_at, ambient in rows:
|
||||
slot = out.get(int(note_id))
|
||||
if slot is None:
|
||||
continue
|
||||
if event == SURFACED and ambient:
|
||||
slot["ambient_count"] = int(n)
|
||||
elif event == SURFACED:
|
||||
slot["surfaced_count"] = int(n)
|
||||
slot["last_surfaced_at"] = iso(last_at)
|
||||
elif event == PULLED:
|
||||
# Pulls are pulls regardless of what surfaced the record — the
|
||||
# question a pull answers ("did anyone ever open this?") doesn't
|
||||
# depend on how it was found.
|
||||
slot["pull_count"] = slot["pull_count"] + int(n)
|
||||
latest = iso(last_at)
|
||||
if latest and (slot["last_pulled_at"] or "") < latest:
|
||||
slot["last_pulled_at"] = latest
|
||||
for note_id, event, n in away_rows:
|
||||
slot = out.get(int(note_id))
|
||||
if slot is None:
|
||||
continue
|
||||
if event == SURFACED:
|
||||
slot["surfaced_away_count"] = int(n)
|
||||
elif event == PULLED:
|
||||
slot["pulled_away_count"] = int(n)
|
||||
return out
|
||||
|
||||
|
||||
def _row_id(row: dict, key: str) -> int | None:
|
||||
"""The note id on a payload row, or None when there is not one to read.
|
||||
|
||||
Skipping is deliberate: an id this cannot parse is not a reason to fail a
|
||||
whole list, and GUESSING one would credit another record's counts to this
|
||||
row — a wrong chip is worse than no chip, because it reads as a
|
||||
measurement. `bool` is excluded explicitly because `int(True)` is 1, which
|
||||
would quietly attach note #1's usage to a row carrying a flag.
|
||||
"""
|
||||
raw = row.get(key)
|
||||
if raw is None or isinstance(raw, bool):
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def attach_usage(rows: Sequence[dict], *, key: str = "id") -> None:
|
||||
"""Add `usage` to every row of a payload a door is about to return.
|
||||
|
||||
The one seam both doors and every record kind share — list routes, detail
|
||||
routes and `/api/knowledge`, the list a person actually browses notes and
|
||||
lessons in. A copy per call site lets the chip reach some record kinds and
|
||||
not others while each door reads fine on its own.
|
||||
|
||||
ONE AGGREGATE FOR THE WHOLE PAGE. `usage_for_notes` is a single GROUP BY
|
||||
over the id set; calling it per row would be N+1 by construction, which is
|
||||
the one shape a list route must not have.
|
||||
|
||||
EVERY ROW GETS THE KEY, zero-filled, so a record predating the table reads
|
||||
as "never surfaced, never pulled" rather than making the UI treat a missing
|
||||
field as a state. `UsageBadge` then renders nothing at all below one
|
||||
surfacing, because "0/0" would look like a verdict where there is only an
|
||||
absence of evidence.
|
||||
|
||||
NO try/except HERE, deliberately — it is not an oversight. The fail-open
|
||||
already lives one layer down: `usage_for_notes` catches its own failure,
|
||||
reports it through `_report_failure("readout")` and returns the zero-filled
|
||||
map, so a broken readout degrades without breaking the list it decorates.
|
||||
Wrapping it again would swallow the REPORT along with the error, and a
|
||||
silently-swallowed readout failure reads as every counter at zero while
|
||||
the writes land fine.
|
||||
|
||||
Mutates in place and returns None: these rows are the payload, not a copy of it.
|
||||
|
||||
A detail payload is just a one-row list — `await attach_usage([data])` —
|
||||
so the single-record doors share this seam rather than keeping a second
|
||||
shape that could drift from it.
|
||||
"""
|
||||
pairs = [(row, _row_id(row, key)) for row in rows]
|
||||
usage = await usage_for_notes([nid for _, nid in pairs if nid is not None])
|
||||
for row, nid in pairs:
|
||||
if nid is not None:
|
||||
row["usage"] = usage.get(nid, empty_usage())
|
||||
@@ -0,0 +1,90 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note_version import NoteVersion
|
||||
|
||||
# Maximum snapshots retained per note.
|
||||
MAX_VERSIONS = 50
|
||||
|
||||
# Minimum seconds between snapshots. Prevents autosave from consuming all
|
||||
# slots — with autosave at 60 s intervals, a 5-minute gate means at most
|
||||
# one snapshot per editing session segment rather than one per save tick.
|
||||
MIN_VERSION_INTERVAL_SECONDS = 300
|
||||
|
||||
|
||||
async def create_version(
|
||||
user_id: int, note_id: int, body: str, title: str, tags: list[str] | None = None
|
||||
) -> NoteVersion | None:
|
||||
async with async_session() as session:
|
||||
# Fetch the most recent snapshot once for both checks.
|
||||
recent = await session.execute(
|
||||
select(NoteVersion)
|
||||
.where(NoteVersion.note_id == note_id, NoteVersion.user_id == user_id)
|
||||
.order_by(NoteVersion.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last = recent.scalars().first()
|
||||
if last is not None:
|
||||
# Skip if content is identical — no change, no snapshot (git semantics).
|
||||
if (
|
||||
last.body == body
|
||||
and last.title == title
|
||||
and sorted(last.tags or []) == sorted(tags or [])
|
||||
):
|
||||
return None
|
||||
# Skip if within the minimum interval to prevent rapid-fire autosave snapshots.
|
||||
last_at = last.created_at
|
||||
if last_at.tzinfo is None:
|
||||
last_at = last_at.replace(tzinfo=timezone.utc)
|
||||
age = (datetime.now(timezone.utc) - last_at).total_seconds()
|
||||
if age < MIN_VERSION_INTERVAL_SECONDS:
|
||||
return None
|
||||
|
||||
version = NoteVersion(note_id=note_id, user_id=user_id, body=body, title=title, tags=tags or [])
|
||||
session.add(version)
|
||||
await session.commit()
|
||||
await session.refresh(version)
|
||||
|
||||
# Prune rolling versions beyond MAX_VERSIONS. Pinned rows
|
||||
# (pin_kind IS NOT NULL) are excluded from both the counted
|
||||
# bucket and the deletion candidate set, so they survive
|
||||
# indefinitely regardless of rolling autosave volume.
|
||||
await session.execute(
|
||||
text("""
|
||||
DELETE FROM note_versions
|
||||
WHERE id IN (
|
||||
SELECT id FROM note_versions
|
||||
WHERE note_id = :note_id
|
||||
AND user_id = :user_id
|
||||
AND pin_kind IS NULL
|
||||
ORDER BY created_at DESC
|
||||
OFFSET :max_versions
|
||||
)
|
||||
""").bindparams(note_id=note_id, user_id=user_id, max_versions=MAX_VERSIONS)
|
||||
)
|
||||
await session.commit()
|
||||
return version
|
||||
|
||||
|
||||
async def list_versions(user_id: int, note_id: int) -> list[NoteVersion]:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(NoteVersion)
|
||||
.where(NoteVersion.note_id == note_id, NoteVersion.user_id == user_id)
|
||||
.order_by(NoteVersion.created_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_version(user_id: int, note_id: int, version_id: int) -> NoteVersion | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(NoteVersion).where(
|
||||
NoteVersion.id == version_id,
|
||||
NoteVersion.note_id == note_id,
|
||||
NoteVersion.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,461 @@
|
||||
"""Notification service for security alerts and task reminders."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete as sa_delete, func, select, text
|
||||
from sqlalchemy import update as sa_update
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.notification import Notification
|
||||
from grimoire.models.user import User
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.services.email import _email_html, is_smtp_configured, send_email
|
||||
from grimoire.services.logging import log_audit
|
||||
from grimoire.services.settings import get_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_notification_task: asyncio.Task | None = None
|
||||
|
||||
SECURITY_EVENT_LABELS = {
|
||||
"login": "New Login",
|
||||
"login_failed": "Failed Login Attempt",
|
||||
"logout": "Logout",
|
||||
"password_change": "Password Changed",
|
||||
}
|
||||
|
||||
|
||||
async def _get_user_notification_pref(user_id: int, key: str) -> bool:
|
||||
"""Check if a user has a notification preference enabled (default True)."""
|
||||
return await get_setting(user_id, key, "true") != "false"
|
||||
|
||||
|
||||
async def _get_user_email(user_id: int) -> str | None:
|
||||
"""Get a user's email address."""
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, user_id)
|
||||
return user.email if user else None
|
||||
|
||||
|
||||
async def notify_security_event(
|
||||
user_id: int, event_type: str, details: dict | None = None
|
||||
) -> None:
|
||||
"""Send a security alert email to a user if they have alerts enabled and an email set."""
|
||||
try:
|
||||
if not await is_smtp_configured():
|
||||
return
|
||||
|
||||
if not await _get_user_notification_pref(user_id, "notify_security_alerts"):
|
||||
return
|
||||
|
||||
email = await _get_user_email(user_id)
|
||||
if not email:
|
||||
return
|
||||
|
||||
label = SECURITY_EVENT_LABELS.get(event_type, event_type)
|
||||
ip = details.get("ip_address", "Unknown") if details else "Unknown"
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
detail_rows = ""
|
||||
if details:
|
||||
for k, v in details.items():
|
||||
if k == "ip_address":
|
||||
continue
|
||||
detail_rows += (
|
||||
f'<tr>'
|
||||
f'<td style="padding:6px 0;color:#6b7280;font-size:13px;width:120px;">{k}</td>'
|
||||
f'<td style="padding:6px 0;color:#111827;font-size:13px;">{v}</td>'
|
||||
f'</tr>'
|
||||
)
|
||||
|
||||
body = f"""
|
||||
<p style="margin:0 0 16px;color:#111827;font-size:15px;font-weight:600;">{label}</p>
|
||||
<table style="width:100%;border-collapse:collapse;background:#f9fafb;border:1px solid #e5e7eb;border-radius:6px;">
|
||||
<tr>
|
||||
<td style="padding:8px 12px;color:#6b7280;font-size:13px;width:120px;border-bottom:1px solid #e5e7eb;">Time</td>
|
||||
<td style="padding:8px 12px;color:#111827;font-size:13px;border-bottom:1px solid #e5e7eb;">{timestamp}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px 12px;color:#6b7280;font-size:13px;width:120px;{';border-bottom:1px solid #e5e7eb' if detail_rows else ''}">IP Address</td>
|
||||
<td style="padding:8px 12px;color:#111827;font-size:13px;{';border-bottom:1px solid #e5e7eb' if detail_rows else ''}">{ip}</td>
|
||||
</tr>
|
||||
{detail_rows}
|
||||
</table>
|
||||
<p style="margin:16px 0 0;color:#9ca3af;font-size:12px;">If this wasn't you, change your password immediately.</p>
|
||||
"""
|
||||
await send_email(email, f"Fabled Grimoire - {label}", _email_html("Security Alert", body))
|
||||
except Exception:
|
||||
logger.exception("Failed to send security notification for user %d", user_id)
|
||||
|
||||
|
||||
async def send_password_reset_email(email: str, reset_url: str) -> None:
|
||||
"""Send a password reset email with a link to reset the user's password."""
|
||||
body = f"""
|
||||
<p style="margin:0 0 16px;color:#374151;font-size:15px;">
|
||||
We received a request to reset your password. Click the button below to choose a new password.
|
||||
</p>
|
||||
<div style="text-align:center;margin:24px 0;">
|
||||
<a href="{reset_url}" style="display:inline-block;background:#7c3aed;color:#ffffff;padding:12px 32px;border-radius:6px;text-decoration:none;font-weight:600;font-size:15px;">
|
||||
Reset Password
|
||||
</a>
|
||||
</div>
|
||||
<p style="margin:0 0 6px;color:#6b7280;font-size:13px;">This link expires in 1 hour.</p>
|
||||
<p style="margin:0 0 16px;color:#6b7280;font-size:13px;">If you didn't request this, you can safely ignore this email.</p>
|
||||
<hr style="border:none;border-top:1px solid #e5e7eb;margin:16px 0;">
|
||||
<p style="margin:0;color:#9ca3af;font-size:12px;">If the button doesn't work, copy and paste this link:</p>
|
||||
<p style="margin:4px 0 0;color:#9ca3af;font-size:12px;word-break:break-all;">{reset_url}</p>
|
||||
"""
|
||||
await send_email(email, "Fabled Grimoire - Password Reset", _email_html("Password Reset", body))
|
||||
|
||||
|
||||
async def send_password_reset_success_email(email: str) -> None:
|
||||
"""Send a notification that the password was successfully reset."""
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
body = f"""
|
||||
<p style="margin:0 0 12px;color:#374151;font-size:15px;">
|
||||
Your password was successfully changed at <strong>{timestamp}</strong>.
|
||||
</p>
|
||||
<p style="margin:0;color:#6b7280;font-size:13px;">If you didn't make this change, please contact your administrator immediately.</p>
|
||||
"""
|
||||
await send_email(email, "Fabled Grimoire - Password Changed", _email_html("Password Changed", body))
|
||||
|
||||
|
||||
async def send_invitation_email(email: str, invite_url: str, invited_by_username: str) -> None:
|
||||
"""Send a branded invitation email with a registration link."""
|
||||
body = f"""
|
||||
<p style="margin:0 0 16px;color:#374151;font-size:15px;">
|
||||
<strong>{invited_by_username}</strong> has invited you to join Fabled Grimoire.
|
||||
Click the button below to create your account.
|
||||
</p>
|
||||
<div style="text-align:center;margin:24px 0;">
|
||||
<a href="{invite_url}" style="display:inline-block;background:#7c3aed;color:#ffffff;padding:12px 32px;border-radius:6px;text-decoration:none;font-weight:600;font-size:15px;">
|
||||
Accept Invitation
|
||||
</a>
|
||||
</div>
|
||||
<p style="margin:0 0 6px;color:#6b7280;font-size:13px;">This invitation expires in 7 days.</p>
|
||||
<p style="margin:0 0 16px;color:#6b7280;font-size:13px;">If you weren't expecting this, you can safely ignore this email.</p>
|
||||
<hr style="border:none;border-top:1px solid #e5e7eb;margin:16px 0;">
|
||||
<p style="margin:0;color:#9ca3af;font-size:12px;">If the button doesn't work, copy and paste this link:</p>
|
||||
<p style="margin:4px 0 0;color:#9ca3af;font-size:12px;word-break:break-all;">{invite_url}</p>
|
||||
"""
|
||||
await send_email(email, "Fabled Grimoire - You're Invited!", _email_html("You're Invited!", body))
|
||||
|
||||
|
||||
def utc_day_start(day: date) -> datetime:
|
||||
"""Midnight UTC on `day`, as an AWARE datetime — the reminder dedup window.
|
||||
|
||||
Deliberately a `datetime` and not the bare `date`. Comparing a
|
||||
`timestamptz` column against a `date` does work, via an implicit cast — but
|
||||
Postgres resolves that cast in the SESSION's TimeZone, so the window would
|
||||
move with a server setting nobody remembers is load-bearing. An aware UTC
|
||||
datetime says what it means and compares the same way everywhere.
|
||||
"""
|
||||
return datetime.combine(day, time.min, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
async def check_due_tasks() -> None:
|
||||
"""Check for tasks due today and send reminder emails."""
|
||||
if not await is_smtp_configured():
|
||||
return
|
||||
|
||||
today = date.today()
|
||||
window_start = utc_day_start(today)
|
||||
|
||||
async with async_session() as session:
|
||||
# Find tasks due today or overdue, not done
|
||||
result = await session.execute(
|
||||
select(Note)
|
||||
.where(
|
||||
Note.status.isnot(None),
|
||||
Note.status != "done",
|
||||
Note.due_date <= today,
|
||||
)
|
||||
)
|
||||
tasks = result.scalars().all()
|
||||
|
||||
if not tasks:
|
||||
return
|
||||
|
||||
# Group by user
|
||||
tasks_by_user: dict[int, list[Note]] = {}
|
||||
for task in tasks:
|
||||
if task.user_id:
|
||||
tasks_by_user.setdefault(task.user_id, []).append(task)
|
||||
|
||||
for user_id, user_tasks in tasks_by_user.items():
|
||||
try:
|
||||
# Check notification pref
|
||||
if not await _get_user_notification_pref(user_id, "notify_task_reminders"):
|
||||
continue
|
||||
|
||||
email = await _get_user_email(user_id)
|
||||
if not email:
|
||||
continue
|
||||
|
||||
# Dedup: check if we already sent a reminder today.
|
||||
# `window_start` is an aware datetime, NOT `today.isoformat()`.
|
||||
# asyncpg binds a str as VARCHAR and Postgres has no
|
||||
# `timestamptz >= text` operator, so a string would raise on
|
||||
# every run — swallowed by the per-user `except` below, leaving
|
||||
# reminders silently never sending.
|
||||
dedup_result = await session.execute(
|
||||
select(func.count(AppLog.id)).where(
|
||||
AppLog.action == "task_reminder",
|
||||
AppLog.user_id == user_id,
|
||||
AppLog.created_at >= window_start,
|
||||
)
|
||||
)
|
||||
if (dedup_result.scalar() or 0) > 0:
|
||||
continue
|
||||
|
||||
# Build task list HTML
|
||||
task_rows = ""
|
||||
for task in user_tasks:
|
||||
overdue = task.due_date < today if task.due_date else False
|
||||
date_color = "#ef4444" if overdue else "#374151"
|
||||
date_label = f'<span style="color: {date_color};">{iso(task.due_date)}</span>' if task.due_date else ""
|
||||
overdue_badge = ' <span style="color:#ef4444;font-weight:600;font-size:11px;">(overdue)</span>' if overdue else ""
|
||||
task_rows += (
|
||||
f'<tr>'
|
||||
f'<td style="padding:8px 12px;border-bottom:1px solid #e5e7eb;color:#111827;font-size:14px;">{task.title}</td>'
|
||||
f'<td style="padding:8px 12px;border-bottom:1px solid #e5e7eb;font-size:14px;">{date_label}{overdue_badge}</td>'
|
||||
f'<td style="padding:8px 12px;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:13px;">{task.status}</td>'
|
||||
f'</tr>'
|
||||
)
|
||||
|
||||
body = f"""
|
||||
<p style="margin:0 0 16px;color:#374151;font-size:15px;">You have <strong>{len(user_tasks)}</strong> task(s) due:</p>
|
||||
<table style="width:100%;border-collapse:collapse;border:1px solid #e5e7eb;border-radius:6px;overflow:hidden;">
|
||||
<tr style="background:#f9fafb;">
|
||||
<th style="padding:8px 12px;text-align:left;font-size:12px;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:0.05em;border-bottom:1px solid #e5e7eb;">Task</th>
|
||||
<th style="padding:8px 12px;text-align:left;font-size:12px;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:0.05em;border-bottom:1px solid #e5e7eb;">Due</th>
|
||||
<th style="padding:8px 12px;text-align:left;font-size:12px;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:0.05em;border-bottom:1px solid #e5e7eb;">Status</th>
|
||||
</tr>
|
||||
{task_rows}
|
||||
</table>
|
||||
"""
|
||||
await send_email(email, f"Fabled Grimoire - {len(user_tasks)} Task(s) Due", _email_html("Task Reminders", body))
|
||||
await log_audit(
|
||||
"task_reminder",
|
||||
user_id=user_id,
|
||||
details={"task_count": len(user_tasks), "task_ids": [t.id for t in user_tasks]},
|
||||
)
|
||||
logger.info("Sent task reminder to user %d (%d tasks)", user_id, len(user_tasks))
|
||||
except Exception:
|
||||
logger.exception("Failed to send task reminder for user %d", user_id)
|
||||
|
||||
|
||||
# Read notifications are kept this long before the hourly sweep deletes them;
|
||||
# unread are kept regardless. Without a sweep the table grows without bound.
|
||||
_NOTIFICATION_RETENTION_DAYS = 30
|
||||
|
||||
|
||||
async def purge_old_read_notifications(retention_days: int = _NOTIFICATION_RETENTION_DAYS) -> int:
|
||||
"""Delete already-read in-app notifications older than retention_days."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
sa_delete(Notification).where(
|
||||
Notification.read_at.isnot(None),
|
||||
Notification.read_at < cutoff,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
async def _notification_tick() -> None:
|
||||
try:
|
||||
await check_due_tasks()
|
||||
except Exception:
|
||||
logger.exception("Error in notification loop")
|
||||
removed = await purge_old_read_notifications()
|
||||
if removed:
|
||||
logger.info("Notification retention: deleted %d read notification(s)", removed)
|
||||
|
||||
|
||||
def start_notification_loop() -> None:
|
||||
global _notification_task
|
||||
if _notification_task is None or _notification_task.done():
|
||||
from grimoire.services.background import start_periodic
|
||||
_notification_task = start_periodic(3600, _notification_tick, label="notifications") # hourly
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# In-app notifications (Notification model — shares, group events)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def create_in_app_notification(user_id: int, notif_type: str, payload: dict):
|
||||
"""Create an in-app Notification record."""
|
||||
async with async_session() as session:
|
||||
n = Notification(user_id=user_id, type=notif_type, payload=payload)
|
||||
session.add(n)
|
||||
await session.commit()
|
||||
await session.refresh(n)
|
||||
return n
|
||||
|
||||
|
||||
async def _fire_share_email(user_id: int, subject: str, body_text: str) -> None:
|
||||
try:
|
||||
if not await is_smtp_configured():
|
||||
return
|
||||
email = await _get_user_email(user_id)
|
||||
if email:
|
||||
html = _email_html(subject, f"<p>{body_text.replace(chr(10), '<br>')}</p>")
|
||||
await send_email(email, subject, html)
|
||||
except Exception:
|
||||
logger.exception("Share email notification failed for user %d", user_id)
|
||||
|
||||
|
||||
async def _group_member_ids(group_id: int) -> list[int]:
|
||||
from grimoire.models.group import GroupMembership
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(GroupMembership.user_id).where(GroupMembership.group_id == group_id)
|
||||
)).scalars().all()
|
||||
return list(rows)
|
||||
|
||||
|
||||
async def notify_project_shared(
|
||||
project_id: int,
|
||||
permission: str,
|
||||
invited_by_user_id: int,
|
||||
target_user_id: int | None,
|
||||
target_group_id: int | None,
|
||||
) -> None:
|
||||
from grimoire.models.project import Project
|
||||
async with async_session() as session:
|
||||
project = await session.get(Project, project_id)
|
||||
inviter = await session.get(User, invited_by_user_id)
|
||||
if not project or not inviter:
|
||||
return
|
||||
|
||||
recipients: list[int] = []
|
||||
if target_user_id:
|
||||
recipients = [target_user_id]
|
||||
elif target_group_id:
|
||||
recipients = [uid for uid in await _group_member_ids(target_group_id) if uid != invited_by_user_id]
|
||||
|
||||
url = f"/projects/{project_id}"
|
||||
msg = f"{inviter.username} shared \"{project.title}\" with you as {permission}"
|
||||
|
||||
for uid in recipients:
|
||||
await create_in_app_notification(uid, "project_shared", {
|
||||
"project_id": project_id,
|
||||
"project_title": project.title,
|
||||
"permission": permission,
|
||||
"invited_by": inviter.username,
|
||||
"url": url,
|
||||
})
|
||||
asyncio.create_task(_fire_share_email(uid, f"[Fabled] {inviter.username} shared a project with you", msg))
|
||||
|
||||
|
||||
async def notify_note_shared(
|
||||
note_id: int,
|
||||
permission: str,
|
||||
invited_by_user_id: int,
|
||||
target_user_id: int | None,
|
||||
target_group_id: int | None,
|
||||
) -> None:
|
||||
from grimoire.models.note import Note
|
||||
async with async_session() as session:
|
||||
note = await session.get(Note, note_id)
|
||||
inviter = await session.get(User, invited_by_user_id)
|
||||
if not note or not inviter:
|
||||
return
|
||||
|
||||
recipients: list[int] = []
|
||||
if target_user_id:
|
||||
recipients = [target_user_id]
|
||||
elif target_group_id:
|
||||
recipients = [uid for uid in await _group_member_ids(target_group_id) if uid != invited_by_user_id]
|
||||
|
||||
route = "tasks" if note.is_task else "notes"
|
||||
url = f"/{route}/{note_id}"
|
||||
msg = f"{inviter.username} shared \"{note.title}\" with you as {permission}"
|
||||
|
||||
for uid in recipients:
|
||||
await create_in_app_notification(uid, "note_shared", {
|
||||
"note_id": note_id,
|
||||
"note_title": note.title,
|
||||
"permission": permission,
|
||||
"invited_by": inviter.username,
|
||||
"url": url,
|
||||
})
|
||||
asyncio.create_task(_fire_share_email(uid, f"[Fabled] {inviter.username} shared a note with you", msg))
|
||||
|
||||
|
||||
async def notify_group_added(
|
||||
group_id: int, role: str, invited_by_user_id: int, target_user_id: int
|
||||
) -> None:
|
||||
from grimoire.models.group import Group
|
||||
async with async_session() as session:
|
||||
group = await session.get(Group, group_id)
|
||||
inviter = await session.get(User, invited_by_user_id)
|
||||
if not group or not inviter:
|
||||
return
|
||||
|
||||
url = f"/groups/{group_id}"
|
||||
msg = f"{inviter.username} added you to group \"{group.name}\" as {role}"
|
||||
|
||||
await create_in_app_notification(target_user_id, "group_added", {
|
||||
"group_id": group_id,
|
||||
"group_name": group.name,
|
||||
"role": role,
|
||||
"invited_by": inviter.username,
|
||||
"url": url,
|
||||
})
|
||||
asyncio.create_task(_fire_share_email(target_user_id, "[Fabled] You've been added to a group", msg))
|
||||
|
||||
|
||||
async def list_in_app_notifications(user_id: int, unread_only: bool = True) -> list[dict]:
|
||||
async with async_session() as session:
|
||||
q = select(Notification).where(Notification.user_id == user_id)
|
||||
if unread_only:
|
||||
q = q.where(Notification.read_at.is_(None))
|
||||
q = q.order_by(Notification.created_at.desc()).limit(50)
|
||||
rows = (await session.execute(q)).scalars().all()
|
||||
return [n.to_dict() for n in rows]
|
||||
|
||||
|
||||
async def unread_notification_count(user_id: int) -> int:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(func.count()).where(
|
||||
Notification.user_id == user_id,
|
||||
Notification.read_at.is_(None),
|
||||
)
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
|
||||
async def mark_notification_read(user_id: int, notification_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
n = (await session.execute(
|
||||
select(Notification).where(
|
||||
Notification.id == notification_id,
|
||||
Notification.user_id == user_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if not n:
|
||||
return False
|
||||
n.read_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def mark_all_notifications_read(user_id: int) -> int:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
sa_update(Notification)
|
||||
.where(Notification.user_id == user_id, Notification.read_at.is_(None))
|
||||
.values(read_at=datetime.now(timezone.utc))
|
||||
.returning(Notification.id)
|
||||
)
|
||||
await session.commit()
|
||||
return len(result.fetchall())
|
||||
@@ -0,0 +1,124 @@
|
||||
import logging
|
||||
import urllib.parse
|
||||
|
||||
import httpx
|
||||
|
||||
from grimoire.config import Config
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.user import User
|
||||
from sqlalchemy import select
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_oidc_config: dict | None = None
|
||||
|
||||
|
||||
async def get_oidc_config() -> dict:
|
||||
global _oidc_config
|
||||
if _oidc_config is not None:
|
||||
return _oidc_config
|
||||
|
||||
discovery_url = Config.OIDC_ISSUER.rstrip("/") + "/.well-known/openid-configuration"
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(discovery_url, follow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
_oidc_config = resp.json()
|
||||
logger.info("OIDC config loaded from %s", discovery_url)
|
||||
return _oidc_config
|
||||
|
||||
|
||||
async def build_auth_url(state: str, code_challenge: str) -> str:
|
||||
oidc = await get_oidc_config()
|
||||
redirect_uri = Config.BASE_URL.rstrip("/") + "/api/auth/oauth/callback"
|
||||
params = {
|
||||
"response_type": "code",
|
||||
"client_id": Config.OIDC_CLIENT_ID,
|
||||
"redirect_uri": redirect_uri,
|
||||
"scope": Config.OIDC_SCOPES,
|
||||
"state": state,
|
||||
"code_challenge": code_challenge,
|
||||
"code_challenge_method": "S256",
|
||||
}
|
||||
return oidc["authorization_endpoint"] + "?" + urllib.parse.urlencode(params)
|
||||
|
||||
|
||||
async def exchange_code(code: str, redirect_uri: str, code_verifier: str) -> dict:
|
||||
oidc = await get_oidc_config()
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
oidc["token_endpoint"],
|
||||
data={
|
||||
"grant_type": "authorization_code",
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"client_id": Config.OIDC_CLIENT_ID,
|
||||
"client_secret": Config.OIDC_CLIENT_SECRET,
|
||||
"code_verifier": code_verifier,
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def get_userinfo(access_token: str) -> dict:
|
||||
oidc = await get_oidc_config()
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
oidc["userinfo_endpoint"],
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def find_or_create_oauth_user(
|
||||
sub: str, email: str, preferred_username: str
|
||||
) -> User:
|
||||
async with async_session() as session:
|
||||
# 1. Look up by oauth_sub
|
||||
result = await session.execute(select(User).where(User.oauth_sub == sub))
|
||||
user = result.scalars().first()
|
||||
if user:
|
||||
return user
|
||||
|
||||
# 2. Look up by email — auto-link
|
||||
if email:
|
||||
from sqlalchemy import func
|
||||
result = await session.execute(
|
||||
select(User).where(func.lower(User.email) == email.lower())
|
||||
)
|
||||
user = result.scalars().first()
|
||||
if user:
|
||||
user.oauth_sub = sub
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
logger.info(
|
||||
"Linked OAuth sub %s to existing user %d (%s) via email",
|
||||
sub, user.id, user.username,
|
||||
)
|
||||
return user
|
||||
|
||||
# 3. Create new user — pick a non-colliding username
|
||||
base_username = preferred_username or (email.split("@")[0] if email else "user")
|
||||
candidate = base_username
|
||||
suffix = 2
|
||||
while True:
|
||||
result = await session.execute(
|
||||
select(User).where(User.username == candidate)
|
||||
)
|
||||
if not result.scalars().first():
|
||||
break
|
||||
candidate = f"{base_username}_{suffix}"
|
||||
suffix += 1
|
||||
|
||||
user = User(
|
||||
username=candidate,
|
||||
email=email or None,
|
||||
password_hash=None,
|
||||
oauth_sub=sub,
|
||||
)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
logger.info("Auto-provisioned OAuth user %d (%s) for sub %s", user.id, user.username, sub)
|
||||
return user
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Where a task sits — its project, its milestone, its step position, what is next.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
An agent reporting finished work to the operator is asked to place it: which
|
||||
milestone, which step of how many, what comes next. Without those facts in
|
||||
hand it reconstructs them from memory, and a reconstruction reads exactly like
|
||||
the real thing while being wrong — an invented milestone title, a "next" step
|
||||
that is already done. So the facts come back on the write that changes a task,
|
||||
where the report is about to be written, instead of being left to recall.
|
||||
|
||||
THE SHAPE
|
||||
|
||||
{"project": {"id", "title"},
|
||||
"milestone": {"id", "title", "status"},
|
||||
"position": {"step": 3, "of": 6},
|
||||
"progress": {"completed", "total", "pct"},
|
||||
"next": {"id", "title", "status"} | None}
|
||||
|
||||
`milestone`, `position`, `progress` and `next` appear only for a task in a
|
||||
milestone; a task with a project and no milestone gets `project` alone; a task
|
||||
with neither gets no placement at all (None), which the doors omit rather than
|
||||
send empty.
|
||||
|
||||
WHAT "STEP" AND "NEXT" MEAN
|
||||
|
||||
Notes carry no order column, so a milestone's steps are in CREATION order
|
||||
(created_at, then id) — the order a plan's steps are written in, and the order
|
||||
a batch create inserts them. Deliberately NOT get_milestone's listing order
|
||||
(status, then last update): that is a display choice, and it reshuffles every
|
||||
time anything is touched.
|
||||
|
||||
`next` is the first open step (todo or in_progress) AFTER this one; when every
|
||||
later step is closed it falls back to the earliest open step before it, since
|
||||
that is still the milestone's next piece of work. None when nothing else is open.
|
||||
|
||||
ACCESS
|
||||
|
||||
Siblings are read through access.readable_notes_clause, so a collaborator on a
|
||||
shared task is never shown the title of a step they cannot open. Position and
|
||||
progress are computed over that same readable set — one list, so the counts can
|
||||
never disagree with the titles they sit beside. For an owner the readable set
|
||||
is every step, and the numbers equal get_milestone's.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.milestone import Milestone
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.services import access as access_svc
|
||||
from grimoire.services.milestones import OPEN_STEP_STATUSES, _progress_from_counts
|
||||
|
||||
# Imported, not restated. The milestone summary names a plan's next open step
|
||||
# too, and the two answers must agree about what "open" means.
|
||||
_OPEN = OPEN_STEP_STATUSES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _next_open(steps: list, current_id: int) -> dict | None:
|
||||
"""The next open step after `current_id`, else the earliest open one before it."""
|
||||
index = next((i for i, s in enumerate(steps) if s.id == current_id), -1)
|
||||
later = [s for s in steps[index + 1:] if s.status in _OPEN]
|
||||
earlier = [s for s in steps[:max(index, 0)] if s.status in _OPEN]
|
||||
pick = (later or earlier or [None])[0]
|
||||
if pick is None:
|
||||
return None
|
||||
return {"id": pick.id, "title": pick.title, "status": pick.status}
|
||||
|
||||
|
||||
async def task_placement(user_id: int, task) -> dict | None:
|
||||
"""Placement for `task` as `user_id` may see it, or None when it has none.
|
||||
|
||||
`task` is the Note the caller already holds — it was just written or read —
|
||||
so its own row is not fetched again.
|
||||
"""
|
||||
project_id = getattr(task, "project_id", None)
|
||||
milestone_id = getattr(task, "milestone_id", None)
|
||||
if not project_id and not milestone_id:
|
||||
return None
|
||||
|
||||
async with async_session() as session:
|
||||
milestone = None
|
||||
if milestone_id:
|
||||
milestone = (await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.id == milestone_id, Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)).scalars().first()
|
||||
project_id = project_id or (milestone.project_id if milestone else None)
|
||||
|
||||
project = None
|
||||
if project_id and await access_svc.can_read_project(user_id, project_id):
|
||||
project = await session.get(Project, project_id)
|
||||
if project is not None and project.deleted_at is not None:
|
||||
project = None
|
||||
|
||||
steps: list = []
|
||||
if milestone is not None:
|
||||
steps = list((await session.execute(
|
||||
select(Note).where(
|
||||
Note.milestone_id == milestone.id,
|
||||
Note.status.isnot(None),
|
||||
Note.deleted_at.is_(None),
|
||||
access_svc.readable_notes_clause(user_id),
|
||||
).order_by(Note.created_at.asc(), Note.id.asc())
|
||||
)).scalars().all())
|
||||
|
||||
out: dict = {}
|
||||
if project is not None:
|
||||
out["project"] = {"id": project.id, "title": project.title}
|
||||
# A milestone is shown only to someone who can read its project (or who
|
||||
# owns it): the task being readable does not make its plan readable.
|
||||
if milestone is not None and (project is not None or milestone.user_id == user_id):
|
||||
counts: dict[str, int] = {}
|
||||
for step in steps:
|
||||
counts[step.status] = counts.get(step.status, 0) + 1
|
||||
progress = _progress_from_counts(counts)
|
||||
position = next((i for i, s in enumerate(steps, start=1) if s.id == task.id), None)
|
||||
out["milestone"] = {"id": milestone.id, "title": milestone.title, "status": milestone.status}
|
||||
out["position"] = {"step": position, "of": len(steps)}
|
||||
out["progress"] = {k: progress[k] for k in ("completed", "total", "pct")}
|
||||
out["next"] = _next_open(steps, task.id)
|
||||
return out or None
|
||||
|
||||
|
||||
async def attach_placement(user_id: int, data: dict, task) -> dict:
|
||||
"""Add `placement` to a task payload the door is about to return.
|
||||
|
||||
Fail-open, like every in-band decoration: the write it rides on has
|
||||
already happened, and a placement lookup that errors must not turn a
|
||||
successful update into a reported failure. Omitted, never sent empty.
|
||||
"""
|
||||
try:
|
||||
placement = await task_placement(user_id, task)
|
||||
except Exception: # noqa: BLE001 - a decoration never breaks its payload
|
||||
logger.warning("placement lookup failed for task %s", getattr(task, "id", None), exc_info=True)
|
||||
return data
|
||||
if placement:
|
||||
data["placement"] = placement
|
||||
return data
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Planning service — start_planning creates a MILESTONE seeded as the plan
|
||||
container, surfacing the project's applicable Rulebook rules at the planning
|
||||
moment so rules land before any work.
|
||||
|
||||
The milestone IS the plan: its `body` holds the design/intent (Goal/Approach/
|
||||
Verification), and the individual steps live as first-class child tasks
|
||||
(milestone_id) rather than checkboxes crammed into one body. The legacy
|
||||
kind=plan task is retired going forward — start_planning never creates one.
|
||||
|
||||
Steps passed up front are created in the SAME transaction as the milestone
|
||||
(services/record_batch.py), so the plan body and the steps can cite each other
|
||||
through `{{ref:N}}` placeholders instead of predicted ids.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from grimoire.services import milestones as milestones_svc
|
||||
from grimoire.services import notes as notes_svc
|
||||
from grimoire.services import projects as projects_svc
|
||||
from grimoire.services import record_batch as batch_svc
|
||||
from grimoire.services import rulebooks as rulebooks_svc
|
||||
from grimoire.services.record_refs import placeholder_keys
|
||||
|
||||
# The plan body template — design only. Steps are NOT checkboxes here; each
|
||||
# step becomes its own child task under this milestone (status, work-logs,
|
||||
# priority of its own).
|
||||
PLAN_TEMPLATE = """## Goal
|
||||
|
||||
## Approach
|
||||
|
||||
## Verification
|
||||
"""
|
||||
|
||||
|
||||
async def start_planning(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
title: str,
|
||||
body: str | None = None,
|
||||
steps: list[batch_svc.BatchItem] | None = None,
|
||||
) -> dict:
|
||||
"""Create a milestone seeded as a plan container and return it with the
|
||||
project's applicable rules + brief context.
|
||||
|
||||
`body` replaces the seeded template; `steps` creates the step-tasks with
|
||||
the milestone, atomically.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"milestone": <milestone dict>,
|
||||
"applicable_rules": [...],
|
||||
"applicable_rules_truncated": bool,
|
||||
"project_rules": [...],
|
||||
"project_goal": str,
|
||||
"open_task_count": int,
|
||||
"steps": [<task dict>, ...], # only when steps were given
|
||||
}
|
||||
"""
|
||||
project = await projects_svc.get_project(user_id, project_id)
|
||||
if project is None:
|
||||
raise ValueError(f"project {project_id} not found")
|
||||
|
||||
plan_body = body or PLAN_TEMPLATE
|
||||
step_notes = []
|
||||
if steps:
|
||||
milestone, step_notes = await batch_svc.create_batch(
|
||||
user_id, steps, project_id=project_id,
|
||||
milestone=batch_svc.BatchMilestone(title=title, body=plan_body),
|
||||
)
|
||||
else:
|
||||
# A placeholder with no batch behind it has nothing to resolve to, and
|
||||
# storing `{{ref:1}}` verbatim would be a reference to nothing.
|
||||
if placeholder_keys(plan_body):
|
||||
raise ValueError(
|
||||
"the plan body uses {{ref:...}} placeholders but no steps were "
|
||||
"given — pass steps=[...] so they have records to name"
|
||||
)
|
||||
milestone = await milestones_svc.create_milestone(
|
||||
user_id,
|
||||
project_id=project_id,
|
||||
title=title,
|
||||
body=plan_body,
|
||||
status="active",
|
||||
)
|
||||
|
||||
applicable = await rulebooks_svc.get_applicable_rules(
|
||||
project_id=project_id, user_id=user_id,
|
||||
)
|
||||
_, open_count = await notes_svc.list_notes(
|
||||
user_id, is_task=True, status="todo", project_id=project_id, limit=1,
|
||||
)
|
||||
|
||||
result = {
|
||||
"milestone": milestone.to_dict(),
|
||||
**rulebooks_svc.rules_payload(applicable, user_id=user_id, source="start_planning"),
|
||||
"project_goal": getattr(project, "goal", "") or "",
|
||||
"open_task_count": open_count,
|
||||
}
|
||||
if steps:
|
||||
result["steps"] = [n.to_dict() for n in step_notes]
|
||||
return result
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Platforms — what a project is built on or ships as, and which projects are
|
||||
which.
|
||||
|
||||
Membership is what makes "is this idea in family for that project?" a lookup
|
||||
rather than a judgment: a family idea is for some platforms, and it reaches
|
||||
every project that is a member of one of them.
|
||||
|
||||
Three ways a project becomes, or refuses to become, a member:
|
||||
|
||||
- **declared** — a person said so: at inception, or in the project's settings.
|
||||
- **detected** — a marker file in a bound repo said so, found by the coverage
|
||||
refresh that already walks the repo archive.
|
||||
- **rejected** — a person said NO. Kept as a row, so the next refresh does not
|
||||
detect it straight back.
|
||||
|
||||
The one invariant everything here protects: **detection only ever ADDS, and
|
||||
only where nobody has answered.** It never overwrites a declared or rejected
|
||||
row, and it never removes anything — not even a detected row whose marker has
|
||||
since disappeared, because membership is what adoption rows hang off and a
|
||||
platform flickering in and out with a repo's file tree would churn a ledger
|
||||
of decisions nobody re-made.
|
||||
|
||||
The catalog itself is global, like the canonical area catalog, and for the
|
||||
same reason writes to it are admin-only: a shared vocabulary anyone can extend
|
||||
stops being shared. Reads are open to any signed-in user.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.family import Platform, ProjectPlatform
|
||||
from grimoire.services import access
|
||||
from grimoire.services.canonical_systems import canonical_slug
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The states that make a project a member. `rejected` is an answer, not
|
||||
# membership.
|
||||
MEMBER_STATES = ("declared", "detected")
|
||||
# What a person may set from a door. `detected` is the refresh's to write.
|
||||
SETTABLE_STATES = ("declared", "rejected")
|
||||
|
||||
|
||||
# --- the catalog -------------------------------------------------------------
|
||||
|
||||
async def list_platforms() -> list[Platform]:
|
||||
"""The whole live catalog, in display order. Global — no owner filter."""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Platform)
|
||||
.where(Platform.deleted_at.is_(None))
|
||||
.order_by(Platform.order_index.asc(), Platform.name.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
def _clean_markers(markers) -> list[str]:
|
||||
"""Markers as stored: stripped, non-empty strings, de-duplicated in order.
|
||||
Anything else is refused by the caller before it gets here."""
|
||||
seen: list[str] = []
|
||||
for m in markers or []:
|
||||
m = str(m).strip()
|
||||
if m and m not in seen:
|
||||
seen.append(m)
|
||||
return seen
|
||||
|
||||
|
||||
def validate_markers(markers) -> str | None:
|
||||
"""The error a markers value would earn, or None. Pure."""
|
||||
if markers is None:
|
||||
return None
|
||||
if not isinstance(markers, list) or not all(isinstance(m, str) for m in markers):
|
||||
return "markers must be a list of glob patterns"
|
||||
if any(m.strip().startswith("/") for m in markers):
|
||||
return "markers are repo-relative — no leading slash"
|
||||
return None
|
||||
|
||||
|
||||
async def create_platform(
|
||||
user_id: int, name: str, *, description: str | None = None,
|
||||
markers: list[str] | None = None,
|
||||
) -> Platform | dict | None:
|
||||
"""Add a platform to the global catalog. Admin only.
|
||||
|
||||
Duplicate-gated on the slug, so "Android App" cannot be added beside
|
||||
"Android app": the existing entry comes back instead of a second spelling
|
||||
of it. None means not permitted, or no usable name.
|
||||
"""
|
||||
if not await access.is_instance_admin(user_id):
|
||||
return None
|
||||
slug = canonical_slug(name)
|
||||
if not slug:
|
||||
return None
|
||||
error = validate_markers(markers)
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
async with async_session() as session:
|
||||
existing = await session.scalar(
|
||||
select(Platform).where(Platform.slug == slug, Platform.deleted_at.is_(None))
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"duplicate": True,
|
||||
"existing_id": existing.id,
|
||||
"message": (
|
||||
f"'{existing.name}' (#{existing.id}) is already this platform — "
|
||||
f"both names reduce to '{slug}'."
|
||||
),
|
||||
}
|
||||
highest = await session.scalar(
|
||||
select(Platform.order_index).order_by(Platform.order_index.desc()).limit(1)
|
||||
)
|
||||
entry = Platform(
|
||||
name=" ".join(name.split()),
|
||||
slug=slug,
|
||||
description=description,
|
||||
markers=_clean_markers(markers),
|
||||
order_index=(highest or 0) + 1,
|
||||
)
|
||||
session.add(entry)
|
||||
await session.commit()
|
||||
await session.refresh(entry)
|
||||
return entry
|
||||
|
||||
|
||||
async def update_platform(user_id: int, platform_id: int, **fields: object) -> Platform | None:
|
||||
"""Rename, re-describe, re-order or re-mark a catalog entry. Admin only.
|
||||
|
||||
A rename recomputes the slug — the display name and the match key must not
|
||||
disagree. Changing the slug of a platform projects already belong to is
|
||||
safe: membership is by id; only a backup carries the slug.
|
||||
"""
|
||||
if not await access.is_instance_admin(user_id):
|
||||
return None
|
||||
if "markers" in fields:
|
||||
error = validate_markers(fields["markers"])
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
async with async_session() as session:
|
||||
entry = await session.get(Platform, platform_id)
|
||||
if entry is None or entry.deleted_at is not None:
|
||||
return None
|
||||
if fields.get("name"):
|
||||
entry.name = " ".join(str(fields["name"]).split())
|
||||
entry.slug = canonical_slug(entry.name)
|
||||
if fields.get("description") is not None:
|
||||
entry.description = fields["description"] or None
|
||||
if fields.get("order_index") is not None:
|
||||
entry.order_index = int(fields["order_index"])
|
||||
if fields.get("markers") is not None:
|
||||
entry.markers = _clean_markers(fields["markers"])
|
||||
entry.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(entry)
|
||||
return entry
|
||||
|
||||
|
||||
async def resolve_slugs(slugs: list[str]) -> tuple[dict[str, int], list[str]]:
|
||||
"""{slug: id} for the slugs the live catalog knows, and the ones it does
|
||||
not. Doors take slugs — readable, stable across installs, and the form an
|
||||
inception record and a backup both carry."""
|
||||
wanted = [s for s in dict.fromkeys(slugs or [])]
|
||||
if not wanted:
|
||||
return {}, []
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(Platform.slug, Platform.id).where(
|
||||
Platform.slug.in_(wanted), Platform.deleted_at.is_(None),
|
||||
)
|
||||
)).all()
|
||||
known = {slug: pid for slug, pid in rows}
|
||||
return known, [s for s in wanted if s not in known]
|
||||
|
||||
|
||||
# --- detection (pure) --------------------------------------------------------
|
||||
|
||||
def marker_matches(marker: str, path: str) -> bool:
|
||||
"""Whether one marker matches one repo-relative path.
|
||||
|
||||
A marker with no slash matches a file's BASENAME anywhere in the tree
|
||||
(`go.mod`, `AndroidManifest.xml`, `vite.config.*`). A marker with a slash
|
||||
matches the whole repo-relative path (`.github/workflows/*`), so a
|
||||
directory-shaped marker cannot be satisfied by a same-named file
|
||||
somewhere else.
|
||||
"""
|
||||
marker = marker.strip()
|
||||
if not marker:
|
||||
return False
|
||||
if "/" in marker:
|
||||
return fnmatch.fnmatchcase(path, marker)
|
||||
return fnmatch.fnmatchcase(path.rsplit("/", 1)[-1], marker)
|
||||
|
||||
|
||||
def detect(catalog: list, paths: list[str]) -> list[int]:
|
||||
"""The ids of every catalog platform at least one of whose markers matches
|
||||
at least one path. Pure, so it is testable against a fixture tree with no
|
||||
forge and no database. A platform with no markers is never detected —
|
||||
that is what declare-only means."""
|
||||
hits: list[int] = []
|
||||
for platform in catalog:
|
||||
markers = list(getattr(platform, "markers", None) or [])
|
||||
if markers and any(marker_matches(m, p) for m in markers for p in paths):
|
||||
hits.append(platform.id)
|
||||
return hits
|
||||
|
||||
|
||||
# --- membership ----------------------------------------------------------------
|
||||
|
||||
async def project_platforms(user_id: int, project_id: int) -> list[dict] | None:
|
||||
"""Every platform the project has an answer for, joined to the catalog.
|
||||
None when the caller cannot read the project.
|
||||
|
||||
Rejected rows are included — a settings screen has to show "no" as well
|
||||
as "yes", or it cannot let anyone change their mind.
|
||||
"""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(ProjectPlatform, Platform)
|
||||
.join(Platform, Platform.id == ProjectPlatform.platform_id)
|
||||
.where(ProjectPlatform.project_id == project_id, Platform.deleted_at.is_(None))
|
||||
.order_by(Platform.order_index.asc(), Platform.name.asc())
|
||||
)).all()
|
||||
return [
|
||||
{"id": p.id, "slug": p.slug, "name": p.name, "state": m.state}
|
||||
for m, p in rows
|
||||
]
|
||||
|
||||
|
||||
def members(platforms: list[dict]) -> list[dict]:
|
||||
"""The rows of `project_platforms` that are membership — the brief form
|
||||
enter_project and get_project carry."""
|
||||
return [
|
||||
{"slug": p["slug"], "name": p["name"], "state": p["state"]}
|
||||
for p in platforms if p["state"] in MEMBER_STATES
|
||||
]
|
||||
|
||||
|
||||
def validate_updates(updates) -> str | None:
|
||||
"""The error a set of membership answers would earn, or None. Pure.
|
||||
|
||||
`updates` is {slug: "declared" | "rejected" | None}. None withdraws the
|
||||
answer — the row goes, and detection may add the platform again later.
|
||||
`detected` is not settable: it is what the refresh writes, and a person
|
||||
claiming it would erase the difference between the two."""
|
||||
if not isinstance(updates, dict):
|
||||
return "platforms must be an object of {slug: state}"
|
||||
for slug, state in updates.items():
|
||||
if not isinstance(slug, str) or not slug:
|
||||
return "each platform is named by its slug"
|
||||
if state is not None and state not in SETTABLE_STATES:
|
||||
return (
|
||||
f"'{state}' is not a state a person sets — use one of "
|
||||
f"{', '.join(SETTABLE_STATES)}, or null to withdraw the answer"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def set_project_platforms(
|
||||
user_id: int, project_id: int, updates: dict[str, str | None],
|
||||
) -> list[dict]:
|
||||
"""Apply a person's answers about a project's platforms. Write-gated on
|
||||
the project. Platforms not named are left exactly as they are.
|
||||
|
||||
Raises ValueError, naming the problem, on a malformed update, an unknown
|
||||
slug, or no write access — before anything is written, so the update
|
||||
applies whole or not at all.
|
||||
"""
|
||||
error = validate_updates(updates)
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
if not await access.can_write_project(user_id, project_id):
|
||||
raise ValueError(f"project {project_id} not found or no write access")
|
||||
known, unknown = await resolve_slugs(list(updates))
|
||||
if unknown:
|
||||
raise ValueError(f"unknown platform(s): {', '.join(unknown)} (list_platforms)")
|
||||
async with async_session() as session:
|
||||
existing = {
|
||||
row.platform_id: row for row in (await session.execute(
|
||||
select(ProjectPlatform).where(ProjectPlatform.project_id == project_id)
|
||||
)).scalars().all()
|
||||
}
|
||||
for slug, state in updates.items():
|
||||
pid = known[slug]
|
||||
row = existing.get(pid)
|
||||
if state is None:
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
elif row is None:
|
||||
session.add(ProjectPlatform(project_id=project_id, platform_id=pid, state=state))
|
||||
else:
|
||||
row.state = state
|
||||
await session.commit()
|
||||
return await project_platforms(user_id, project_id) or []
|
||||
|
||||
|
||||
async def record_detected(project_id: int, platform_ids: list[int]) -> list[int]:
|
||||
"""Add `detected` membership for each platform the project has NO answer
|
||||
for. Returns the ids actually added.
|
||||
|
||||
The refresh's writer, so it takes no user: it runs on the owner's behalf
|
||||
inside a sync the owner's keyring authorised. It never updates or deletes
|
||||
a row — see the module docstring for why that is the whole contract.
|
||||
"""
|
||||
if not platform_ids:
|
||||
return []
|
||||
async with async_session() as session:
|
||||
answered = set((await session.execute(
|
||||
select(ProjectPlatform.platform_id).where(
|
||||
ProjectPlatform.project_id == project_id,
|
||||
)
|
||||
)).scalars().all())
|
||||
added = [pid for pid in dict.fromkeys(platform_ids) if pid not in answered]
|
||||
for pid in added:
|
||||
session.add(ProjectPlatform(project_id=project_id, platform_id=pid, state="detected"))
|
||||
await session.commit()
|
||||
return added
|
||||
|
||||
|
||||
async def detect_for_project(project_id: int, paths: list[str]) -> list[int]:
|
||||
"""Run detection over a refresh's paths and record what is new. The
|
||||
coverage refresh's single call — it must not be able to fail that
|
||||
refresh, so the caller wraps it."""
|
||||
hits = detect(await list_platforms(), paths)
|
||||
added = await record_detected(project_id, hits)
|
||||
if added:
|
||||
logger.info("project %s: detected platform(s) %s", project_id, added)
|
||||
return added
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,284 @@
|
||||
"""Project management service."""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project, ProjectStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _validate_status(status: str) -> str:
|
||||
"""Coerce/validate a project status against ProjectStatus.
|
||||
|
||||
Canonical gate so the MCP create/update_project path (which passes status
|
||||
straight through) can't persist an out-of-enum value — there's no DB CHECK.
|
||||
"""
|
||||
try:
|
||||
return ProjectStatus(status).value
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"Invalid status: {status!r}. Must be one of: {[s.value for s in ProjectStatus]}"
|
||||
)
|
||||
|
||||
|
||||
async def create_project(
|
||||
user_id: int,
|
||||
title: str,
|
||||
description: str = "",
|
||||
goal: str = "",
|
||||
color: str | None = None,
|
||||
status: str = "active",
|
||||
) -> Project:
|
||||
status = _validate_status(status)
|
||||
async with async_session() as session:
|
||||
project = Project(
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
description=description,
|
||||
goal=goal,
|
||||
color=color,
|
||||
status=status,
|
||||
)
|
||||
session.add(project)
|
||||
await session.commit()
|
||||
await session.refresh(project)
|
||||
return project
|
||||
|
||||
|
||||
async def get_project(user_id: int, project_id: int) -> Project | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Project).where(
|
||||
Project.id == project_id, Project.user_id == user_id,
|
||||
Project.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def get_project_by_title(user_id: int, title: str) -> Project | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Project).where(
|
||||
Project.user_id == user_id,
|
||||
func.lower(Project.title) == func.lower(title.strip()),
|
||||
Project.deleted_at.is_(None),
|
||||
).limit(1)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def get_or_create_project(user_id: int, title: str) -> Project:
|
||||
project = await get_project_by_title(user_id, title)
|
||||
if project:
|
||||
return project
|
||||
return await create_project(user_id, title=title)
|
||||
|
||||
|
||||
async def list_projects(user_id: int, status: str | None = None) -> list[Project]:
|
||||
async with async_session() as session:
|
||||
query = select(Project).where(
|
||||
Project.user_id == user_id, Project.deleted_at.is_(None)
|
||||
)
|
||||
if status:
|
||||
query = query.where(Project.status == status)
|
||||
query = query.order_by(Project.updated_at.desc())
|
||||
result = await session.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Project).where(
|
||||
Project.id == project_id, Project.user_id == user_id,
|
||||
Project.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
project = result.scalars().first()
|
||||
if project is None:
|
||||
return None
|
||||
if "status" in fields and fields["status"] is not None:
|
||||
fields["status"] = _validate_status(fields["status"])
|
||||
for key, value in fields.items():
|
||||
if hasattr(project, key):
|
||||
setattr(project, key, value)
|
||||
project.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(project)
|
||||
return project
|
||||
|
||||
|
||||
async def delete_project(user_id: int, project_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Project).where(Project.id == project_id, Project.user_id == user_id)
|
||||
)
|
||||
project = result.scalars().first()
|
||||
if project is None:
|
||||
return False
|
||||
# Unlink notes (cascade handled by DB ON DELETE SET NULL, but we delete project here)
|
||||
await session.delete(project)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def get_project_summaries(
|
||||
user_id: int, project_ids: list[int]
|
||||
) -> dict[int, dict]:
|
||||
"""Summaries for MANY projects — four queries and one session, total.
|
||||
|
||||
Not an `asyncio.gather` over the per-project version below: that is a
|
||||
nested fan-out — each project opens its own session for three queries,
|
||||
then calls the milestone summary, which opens one more per milestone. For
|
||||
a few dozen projects that asks for far more pooled connections than the
|
||||
pool holds, so most of them sit out the checkout timeout and everything
|
||||
else on the instance queues behind them, unrelated routes included.
|
||||
|
||||
Moving an N+1 from the browser to the server does not remove it; it moves
|
||||
it onto the connection pool, where it is worse — the browser at least
|
||||
serialises its calls.
|
||||
"""
|
||||
if not project_ids:
|
||||
return {}
|
||||
|
||||
async with async_session() as session:
|
||||
task_rows = await session.execute(
|
||||
select(Note.project_id, Note.status, func.count(Note.id))
|
||||
.where(
|
||||
Note.user_id == user_id,
|
||||
Note.project_id.in_(project_ids),
|
||||
Note.status.isnot(None),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
.group_by(Note.project_id, Note.status)
|
||||
)
|
||||
task_counts: dict[int, dict[str, int]] = {}
|
||||
for project_id, status, count in task_rows.fetchall():
|
||||
task_counts.setdefault(project_id, {})[status] = count
|
||||
|
||||
note_rows = await session.execute(
|
||||
select(Note.project_id, func.count(Note.id))
|
||||
.where(
|
||||
Note.user_id == user_id,
|
||||
Note.project_id.in_(project_ids),
|
||||
Note.status.is_(None),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
.group_by(Note.project_id)
|
||||
)
|
||||
note_counts = {pid: count for pid, count in note_rows.fetchall()}
|
||||
|
||||
# Deliberately NOT filtered by deleted_at, matching the per-project
|
||||
# version: "last activity" includes trashing something.
|
||||
activity_rows = await session.execute(
|
||||
select(Note.project_id, func.max(Note.updated_at))
|
||||
.where(Note.user_id == user_id, Note.project_id.in_(project_ids))
|
||||
.group_by(Note.project_id)
|
||||
)
|
||||
last_activity = {pid: ts for pid, ts in activity_rows.fetchall()}
|
||||
|
||||
from grimoire.services.milestones import get_project_milestone_summaries
|
||||
milestones = await get_project_milestone_summaries(user_id, project_ids)
|
||||
|
||||
return {
|
||||
pid: {
|
||||
# All three lifecycle keys present so consumers can sum without
|
||||
# `?? 0` guards — the frontend declares them required, and
|
||||
# `undefined + N` renders as NaN.
|
||||
"task_counts": {
|
||||
"todo": 0, "in_progress": 0, "done": 0,
|
||||
**task_counts.get(pid, {}),
|
||||
},
|
||||
"note_count": note_counts.get(pid, 0),
|
||||
"last_activity": (
|
||||
last_activity[pid].isoformat() if last_activity.get(pid) else None
|
||||
),
|
||||
"milestone_summary": milestones.get(pid, []),
|
||||
}
|
||||
for pid in project_ids
|
||||
}
|
||||
|
||||
|
||||
async def get_project_summary(user_id: int, project_id: int) -> dict:
|
||||
"""Return task counts by status, note count, and last activity — the
|
||||
one-project view of get_project_summaries (one rule, not two copies)."""
|
||||
return (await get_project_summaries(user_id, [project_id]))[project_id]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared-access variants (honour ProjectShare in addition to ownership)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def get_project_for_user(
|
||||
accessing_user_id: int, project_id: int
|
||||
) -> tuple[Project, str] | None:
|
||||
"""Returns (project, permission) if user has any access, else None."""
|
||||
from grimoire.services.access import get_project_permission
|
||||
perm = await get_project_permission(accessing_user_id, project_id)
|
||||
if perm is None:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
project = await session.get(Project, project_id)
|
||||
return (project, perm) if project else None
|
||||
|
||||
|
||||
async def list_projects_for_user(user_id: int, status: str | None = None) -> list[dict]:
|
||||
"""Owned projects + shared projects, each dict has 'permission' field."""
|
||||
from grimoire.models.group import GroupMembership
|
||||
from grimoire.models.share import ProjectShare
|
||||
owned = await list_projects(user_id, status)
|
||||
owned_ids = {p.id for p in owned}
|
||||
|
||||
result = []
|
||||
for p in owned:
|
||||
d = p.to_dict() if hasattr(p, "to_dict") else {"id": p.id, "title": p.title}
|
||||
d["permission"] = "owner"
|
||||
result.append(d)
|
||||
|
||||
async with async_session() as session:
|
||||
user_group_ids = (await session.execute(
|
||||
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
||||
)).scalars().all()
|
||||
|
||||
shared_direct = (await session.execute(
|
||||
select(ProjectShare).where(ProjectShare.shared_with_user_id == user_id)
|
||||
)).scalars().all()
|
||||
|
||||
shared_group: list[ProjectShare] = []
|
||||
if user_group_ids:
|
||||
shared_group = (await session.execute(
|
||||
select(ProjectShare).where(
|
||||
ProjectShare.shared_with_group_id.in_(user_group_ids)
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
from grimoire.services.sharing import best_permission_by
|
||||
seen = {
|
||||
pid: perm
|
||||
for pid, perm in best_permission_by(
|
||||
list(shared_direct) + list(shared_group), "project_id"
|
||||
).items()
|
||||
if pid not in owned_ids
|
||||
}
|
||||
|
||||
for pid, perm in seen.items():
|
||||
if status:
|
||||
async with async_session() as session:
|
||||
proj = await session.get(Project, pid)
|
||||
if not proj or proj.status != status:
|
||||
continue
|
||||
else:
|
||||
async with async_session() as session:
|
||||
proj = await session.get(Project, pid)
|
||||
if proj:
|
||||
d = proj.to_dict() if hasattr(proj, "to_dict") else {"id": proj.id, "title": proj.title}
|
||||
d["permission"] = perm
|
||||
d["is_shared"] = True
|
||||
result.append(d)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Create several records at once, so they can cite each other without guessing ids.
|
||||
|
||||
Why this exists is in services/record_refs.py: a session that creates records
|
||||
one by one and predicts the ids of the ones not yet made loses those numbers
|
||||
to any concurrent create, from any session or any user.
|
||||
|
||||
Here the whole batch is ONE transaction: insert every record, flush so the
|
||||
sequence assigns the real ids, rewrite each `{{ref:N}}` with the id and title
|
||||
it names, commit. Other sessions keep creating throughout and cannot interfere
|
||||
— the sequence never hands out the same number twice, so another session's
|
||||
insert just takes a different one. The ids a batch receives are therefore NOT
|
||||
guaranteed consecutive, and nothing here needs them to be: the placeholders
|
||||
are filled with whatever ids came back. Forcing consecutive ids would take a
|
||||
table lock that stalls every user's writes; nobody needs that.
|
||||
|
||||
All or nothing: a bad placeholder, an invalid status, or a failure mid-insert
|
||||
leaves no partial batch behind — which is also why there are no "reserved"
|
||||
stub records to clean up after a session that dies.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.milestone import Milestone
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.services import access as access_svc
|
||||
from grimoire.services import milestones as milestones_svc
|
||||
from grimoire.services import notes as notes_svc
|
||||
from grimoire.services import systems as systems_svc
|
||||
from grimoire.services.record_refs import placeholder_keys, resolve_placeholders
|
||||
|
||||
# A batch is a plan's steps or a handful of linked records, and every item runs
|
||||
# the near-duplicate gate (an embedding search) before the transaction. 50 is
|
||||
# far past any real plan and keeps one call from becoming a bulk import, which
|
||||
# is a different job with a different door.
|
||||
MAX_BATCH = 50
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchItem:
|
||||
"""One record in a batch. `is_task` False makes it a plain note."""
|
||||
|
||||
title: str
|
||||
body: str = ""
|
||||
is_task: bool = True
|
||||
status: str = "todo"
|
||||
priority: str | None = None
|
||||
task_kind: str = "work"
|
||||
tags: list[str] = field(default_factory=list)
|
||||
system_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchMilestone:
|
||||
"""The milestone start_planning creates in the same transaction as its steps."""
|
||||
|
||||
title: str
|
||||
body: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
def check_batch(items: list[BatchItem], milestone: BatchMilestone | None = None) -> None:
|
||||
"""Refuse a batch that cannot be written in full, before any write.
|
||||
|
||||
Every placeholder must name something the batch will actually create;
|
||||
otherwise the rewrite has nothing to put there, and the alternatives —
|
||||
leaving `{{ref:9}}` in a stored body, or dropping it — both store a
|
||||
reference that points nowhere.
|
||||
"""
|
||||
if not items and milestone is None:
|
||||
raise ValueError("a batch needs at least one record")
|
||||
if len(items) > MAX_BATCH:
|
||||
raise ValueError(f"a batch holds at most {MAX_BATCH} records; got {len(items)}")
|
||||
for i, item in enumerate(items, start=1):
|
||||
if not (item.title or "").strip():
|
||||
raise ValueError(f"record {i} has no title")
|
||||
valid = {str(i) for i in range(1, len(items) + 1)}
|
||||
if milestone is not None:
|
||||
valid.add("milestone")
|
||||
texts = [item.body for item in items] + ([milestone.body] if milestone else [])
|
||||
unknown = placeholder_keys(*texts) - valid
|
||||
if unknown:
|
||||
named = ", ".join("{{ref:%s}}" % k for k in sorted(unknown))
|
||||
raise ValueError(
|
||||
f"{named} names no record in this batch. {{{{ref:N}}}} is the Nth "
|
||||
f"record listed (1 to {len(items)})"
|
||||
+ (", and {{ref:milestone}} is the milestone being created." if milestone else ".")
|
||||
)
|
||||
|
||||
|
||||
async def create_batch(
|
||||
user_id: int,
|
||||
items: list[BatchItem],
|
||||
*,
|
||||
project_id: int | None = None,
|
||||
milestone_id: int | None = None,
|
||||
milestone: BatchMilestone | None = None,
|
||||
) -> tuple[Milestone | None, list[Note]]:
|
||||
"""Create `items` (and optionally a new milestone they belong to) atomically.
|
||||
|
||||
`milestone_id` files the items under an EXISTING milestone the caller owns;
|
||||
`milestone` creates a new one alongside them and files them there. Passing
|
||||
both is refused. Returns (the new milestone or None, the notes in input
|
||||
order). Near-duplicate gating is the door's job, exactly as for a single
|
||||
create.
|
||||
"""
|
||||
if milestone is not None and milestone_id:
|
||||
raise ValueError("pass milestone_id (an existing milestone) or a new milestone, not both")
|
||||
check_batch(items, milestone)
|
||||
|
||||
if milestone_id:
|
||||
async with async_session() as session:
|
||||
existing = (await session.execute(
|
||||
select(Milestone).where(
|
||||
Milestone.id == milestone_id, Milestone.deleted_at.is_(None),
|
||||
)
|
||||
)).scalars().first()
|
||||
if existing is None:
|
||||
raise ValueError(f"milestone {milestone_id} not found")
|
||||
if project_id and project_id != existing.project_id:
|
||||
raise ValueError(
|
||||
f"milestone {milestone_id} belongs to project {existing.project_id}, not {project_id}"
|
||||
)
|
||||
project_id = existing.project_id
|
||||
if milestone is not None and not project_id:
|
||||
raise ValueError("a new milestone needs a project_id")
|
||||
# Share-aware: a collaborator with write access to a shared
|
||||
# project may plan in it; a bare owner filter would refuse them.
|
||||
if project_id and not await access_svc.can_write_project(user_id, project_id):
|
||||
raise ValueError(f"project {project_id} not found")
|
||||
|
||||
async with async_session() as session:
|
||||
# Validate every record before adding any: build_note raises on a bad
|
||||
# status/priority, and raising here writes nothing.
|
||||
notes = [
|
||||
notes_svc.build_note(
|
||||
user_id,
|
||||
title=item.title,
|
||||
body=item.body,
|
||||
tags=item.tags,
|
||||
project_id=project_id,
|
||||
milestone_id=milestone_id,
|
||||
status=item.status if item.is_task else None,
|
||||
priority=item.priority if item.is_task else None,
|
||||
task_kind=notes_svc.minted_kind(item.task_kind) if item.is_task else "work",
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
|
||||
new_ms = None
|
||||
if milestone is not None:
|
||||
new_ms = Milestone(
|
||||
user_id=user_id, project_id=project_id, title=milestone.title,
|
||||
description=milestone.description, body=milestone.body, status="active",
|
||||
)
|
||||
session.add(new_ms)
|
||||
await session.flush()
|
||||
for note in notes:
|
||||
note.milestone_id = new_ms.id
|
||||
|
||||
session.add_all(notes)
|
||||
# The flush is where the sequence assigns ids. Nothing is visible to
|
||||
# any other session until the commit below, and if anything between
|
||||
# here and there raises, the context manager rolls it all back.
|
||||
await session.flush()
|
||||
|
||||
refs = {str(i): f'#{n.id} "{n.title}"' for i, n in enumerate(notes, start=1)}
|
||||
if new_ms is not None:
|
||||
refs["milestone"] = f'milestone {new_ms.id} "{new_ms.title}"'
|
||||
new_ms.body = resolve_placeholders(new_ms.body, refs)
|
||||
for note in notes:
|
||||
note.body = resolve_placeholders(note.body, refs)
|
||||
|
||||
await session.commit()
|
||||
for note in notes:
|
||||
await session.refresh(note)
|
||||
if new_ms is not None:
|
||||
await session.refresh(new_ms)
|
||||
|
||||
# After the commit, as a single create does: embedding and System tags are
|
||||
# enrichment on records that now exist, and a failure in either must not
|
||||
# un-create them.
|
||||
if new_ms is not None:
|
||||
milestones_svc.embed_milestone(new_ms)
|
||||
for note, item in zip(notes, items):
|
||||
notes_svc.embed_note(note)
|
||||
if item.system_ids:
|
||||
await systems_svc.set_record_systems(user_id, note.id, item.system_ids)
|
||||
if project_id is not None:
|
||||
await notes_svc._maybe_reactivate_project(project_id)
|
||||
|
||||
return new_ms, notes
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Record references written into prose: refusing guessed ids, resolving placeholders, reading the ids a message names.
|
||||
|
||||
THE DEFECT THIS EXISTS FOR
|
||||
|
||||
A session about to create several records guesses the ids they will get — the
|
||||
last id it saw, plus one, plus two — and writes those guesses into a plan body
|
||||
or a reference note before the records exist. The database never collides:
|
||||
ids come from one Postgres sequence per table, handed out once each. But that
|
||||
sequence is shared by every session and every user, so any concurrent create
|
||||
takes a guessed number, and the reference silently points at someone else's
|
||||
record. More users make it worse, never better.
|
||||
|
||||
Two halves, and they are not alternatives:
|
||||
|
||||
- `{{ref:N}}` PLACEHOLDERS (services/record_batch.py) remove the reason to
|
||||
guess. A batch create inserts its records, receives their real ids and
|
||||
rewrites the placeholders inside ONE transaction, so there is no moment at
|
||||
which an id has to be predicted.
|
||||
- REFUSING A GUESSED ID is the backstop for a session that guesses anyway. It
|
||||
is a refusal, not a warning, because a warning is an instruction and
|
||||
instructions are not always followed.
|
||||
|
||||
HOW A GUESS IS RECOGNISED
|
||||
|
||||
A guess is always just ABOVE the highest id that exists — that is the only
|
||||
place "the next id" can be. So a `#N` is refused when it sits in
|
||||
(max_id, max_id + GUESS_WINDOW]. Numbers below the max are real records (or
|
||||
were), and numbers far above it are not Grimoire ids at all — a PR, a forge
|
||||
issue — so both pass. What this cannot catch, by construction: a guess that
|
||||
another session has ALREADY used. That reference points at a real record, and
|
||||
nothing in the text distinguishes it from a deliberate one. The placeholders
|
||||
are the fix for that case; this is not.
|
||||
|
||||
WHERE IT IS ENFORCED
|
||||
|
||||
At the MCP door (the create/update tools), not in the services. The defect is
|
||||
an agent predicting ids; a person typing `#123` in the web editor has read that
|
||||
number off a record. Enforcing it in `notes.create_note` would also reach
|
||||
restores and internal writers that legitimately carry numbers from elsewhere.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
|
||||
# `#123` as a standalone token. The lookbehind keeps out `{` entities,
|
||||
# URL fragments (`page#12`) and doubled `##`; `\b` after the digits keeps out a
|
||||
# hex colour like `#123abc`.
|
||||
REF_RE = re.compile(r"(?<![\w&#/])#(\d+)\b")
|
||||
|
||||
# How far above the highest existing id a number still reads as a guess. It
|
||||
# only has to cover how far ahead a session predicts, which is the size of the
|
||||
# batch it is planning — a plan's steps, a handful of cross-linked notes. 50 is
|
||||
# generous for that and still narrow enough that an unrelated number (a PR, an
|
||||
# issue on a forge) landing inside it is rare. Widening it trades exactly that.
|
||||
GUESS_WINDOW = 50
|
||||
|
||||
# `{{ref:3}}` names the third record of the same batch; `{{ref:milestone}}` names
|
||||
# the milestone a start_planning call creates alongside its steps.
|
||||
PLACEHOLDER_RE = re.compile(r"\{\{\s*ref:\s*(\d+|milestone)\s*\}\}")
|
||||
|
||||
|
||||
def cited_ids(*texts: str | None) -> set[int]:
|
||||
"""Every `#N` written in these texts."""
|
||||
found: set[int] = set()
|
||||
for text in texts:
|
||||
if text:
|
||||
found.update(int(m) for m in REF_RE.findall(text))
|
||||
return found
|
||||
|
||||
|
||||
async def _max_note_id() -> int:
|
||||
# Across ALL users and including trashed rows: the sequence is global, so
|
||||
# "the next id" is global too. Only the number is read, never a row.
|
||||
async with async_session() as session:
|
||||
return (await session.execute(select(func.max(Note.id)))).scalar() or 0
|
||||
|
||||
|
||||
async def guessed_ids(*texts: str | None) -> list[int]:
|
||||
"""The `#N` references in these texts that can only be predictions."""
|
||||
cited = cited_ids(*texts)
|
||||
if not cited:
|
||||
return []
|
||||
ceiling = await _max_note_id()
|
||||
return sorted(n for n in cited if ceiling < n <= ceiling + GUESS_WINDOW)
|
||||
|
||||
|
||||
async def refuse_guessed_ids(*texts: str | None) -> None:
|
||||
"""Raise if any text cites a note/task id that has not been assigned yet.
|
||||
|
||||
Raised before anything is written, so the caller's create or update does
|
||||
not happen. The message names the fix, because the agent reading it is
|
||||
mid-turn and can apply it immediately.
|
||||
"""
|
||||
guessed = await guessed_ids(*texts)
|
||||
if not guessed:
|
||||
return
|
||||
listed = ", ".join(f"#{n}" for n in guessed)
|
||||
raise ValueError(
|
||||
f"{listed} does not exist yet — it reads as a guessed id, and a guess "
|
||||
"is taken by whatever any other session creates next. Nothing was "
|
||||
"written. An id exists only once a create call RETURNS it: create the "
|
||||
"records first and cite the ids they come back with, or create them "
|
||||
"together with create_records / start_planning(steps=...) and write "
|
||||
"{{ref:N}} where the Nth record's id belongs. If this number is not a "
|
||||
"Grimoire record (a PR, an issue elsewhere), write it without the '#'."
|
||||
)
|
||||
|
||||
|
||||
def placeholder_keys(*texts: str | None) -> set[str]:
|
||||
"""Every `{{ref:...}}` key in these texts: "1", "2", …, "milestone"."""
|
||||
keys: set[str] = set()
|
||||
for text in texts:
|
||||
if text:
|
||||
keys.update(PLACEHOLDER_RE.findall(text))
|
||||
return keys
|
||||
|
||||
|
||||
def resolve_placeholders(text: str | None, refs: dict[str, str]) -> str | None:
|
||||
"""Rewrite `{{ref:...}}` using `refs` (key -> rendered reference).
|
||||
|
||||
Callers validate the keys first; an unknown key here is a bug in the
|
||||
caller, so it raises rather than leaving a placeholder in a stored record.
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
return PLACEHOLDER_RE.sub(lambda m: refs[m.group(1)], text)
|
||||
|
||||
|
||||
# ── Records an operator names in a message ─────────────────────────────────
|
||||
#
|
||||
# "yes go ahead with 4448" says exactly which record is meant, and a ranked
|
||||
# menu cannot use it: a number carries no meaning to an embedding, so the menu
|
||||
# fills with whatever resembles the words AROUND it. These ids are looked up
|
||||
# directly instead (plugin_context.build_autoinject_hint).
|
||||
#
|
||||
# A guard against a false READ, not a false lookup. A number that is not a
|
||||
# record id still has to resolve to a record this user can open before anything
|
||||
# is shown, and the line says it was found by number — but every wrong id is a
|
||||
# record dropped in front of the reader for no reason, so the parser takes a
|
||||
# number only where the sentence makes it a reference.
|
||||
|
||||
# How many named records one message can bring in. A real message names one or
|
||||
# two; past a handful the numbers are a pasted list or a log, not a reference.
|
||||
NAMED_LIMIT = 5
|
||||
|
||||
# A number, with its `#` when it has one. The lookbehind keeps out a number
|
||||
# inside a word, a path or URL (`pulls/198`), a version or date (`1.2`,
|
||||
# `2026-10-03`), money and entities; the lookahead keeps out the same on the
|
||||
# other side, a unit glued on (`300ms`, `40%`) and a thousands separator.
|
||||
_NAMED_RE = re.compile(
|
||||
r"(?<![\w#&$€£.:/\\=@~-])(#?)(\d{1,7})(?![\w%]|[.:/-]\d|,\d{3}(?!\d))"
|
||||
)
|
||||
|
||||
# Words after which `#N` belongs to a different numbering: a forge's (a PR, a
|
||||
# CI run, a job) or one of Grimoire's own id spaces that is not a note's. Rules,
|
||||
# milestones, Systems, projects and work-logs each draw from their own
|
||||
# sequence, so "rule #153" is not note 153 — and note 153 is somebody's record.
|
||||
_OTHER_SPACE = frozenset({
|
||||
"pr", "prs", "pull", "pulls", "request", "requests", "mr", "mrs",
|
||||
"run", "runs", "ci", "build", "builds", "job", "jobs", "pipeline",
|
||||
"action", "actions", "commit", "github", "gitlab", "gitea", "gh",
|
||||
"rule", "rules", "preference", "preferences", "milestone", "milestones",
|
||||
"system", "systems", "project", "projects", "log", "logs", "topic",
|
||||
"topics", "rulebook", "rulebooks", "step", "steps", "rank", "line",
|
||||
"lines", "page", "item", "option", "version", "port",
|
||||
})
|
||||
|
||||
# Words after which a BARE number is a reference. Bare numbers are most of what
|
||||
# a message holds — sizes, counts, durations — so a bare one is taken only
|
||||
# after a word that points at something, at the very start of the message, or
|
||||
# continuing a list that started as a reference.
|
||||
_REF_WORDS = frozenset({
|
||||
"with", "for", "on", "at", "about", "re", "regarding", "into", "up",
|
||||
"task", "tasks", "issue", "issues", "note", "notes", "record", "records",
|
||||
"snippet", "snippets", "lesson", "lessons", "spike", "spikes", "process",
|
||||
"continue", "resume", "start", "reopen", "open", "close", "finish",
|
||||
"see", "check", "read", "do", "id", "ids",
|
||||
})
|
||||
|
||||
# A word after a bare number that makes it a quantity: "for 300 seconds",
|
||||
# "about 500 records". Not applied after `#`, which is never a quantity.
|
||||
_UNITS = frozenset({
|
||||
"s", "sec", "secs", "second", "seconds", "ms", "min", "mins", "minute",
|
||||
"minutes", "h", "hr", "hrs", "hour", "hours", "day", "days", "week",
|
||||
"weeks", "month", "months", "year", "years", "char", "chars", "character",
|
||||
"characters", "token", "tokens", "line", "lines", "word", "words", "byte",
|
||||
"bytes", "kb", "mb", "gb", "px", "row", "rows", "record", "records",
|
||||
"note", "notes", "task", "tasks", "item", "items", "file", "files",
|
||||
"test", "tests", "call", "calls", "time", "times", "x", "percent",
|
||||
"result", "results", "hit", "hits", "query", "queries", "message",
|
||||
"messages", "prompt", "prompts", "step", "steps", "point", "points",
|
||||
})
|
||||
|
||||
_FENCE_RE = re.compile(r"```.*?(?:```|\Z)", re.S)
|
||||
_PREV_WORD_RE = re.compile(r"([A-Za-z]+)[\s\"'(\[*_:]*\Z")
|
||||
_NEXT_WORD_RE = re.compile(r"\s*([A-Za-z]+)")
|
||||
_OPENS_RE = re.compile(r"[\s\"'(\[*_>-]*")
|
||||
_LIST_JOIN_RE = re.compile(r"\s*(?:,|,?\s*(?:and|or|&|\+|/))\s*", re.I)
|
||||
|
||||
|
||||
def _prev_word(before: str) -> str:
|
||||
"""The word right before a number, past a space, a quote or a bracket."""
|
||||
m = _PREV_WORD_RE.search(before)
|
||||
return m.group(1).lower() if m else ""
|
||||
|
||||
|
||||
def _next_word(after: str) -> str:
|
||||
m = _NEXT_WORD_RE.match(after)
|
||||
return m.group(1).lower() if m else ""
|
||||
|
||||
|
||||
def named_record_ids(prompt: str | None, limit: int = NAMED_LIMIT) -> list[int]:
|
||||
"""The record ids a message names, in the order it names them.
|
||||
|
||||
`#4448` unless the word before it is another numbering ("PR #198"); a bare
|
||||
number of three or more digits when it opens the message ("4417 sounds
|
||||
right"), follows a word that points at something ("go ahead with 4448") or
|
||||
continues a list one of those started ("4448, 4449 and 4450"), and is not
|
||||
a quantity ("for 300 seconds"). Fenced code is skipped: a pasted log is
|
||||
full of numbers and names nothing.
|
||||
|
||||
Read the operator's OWN words, never a query enriched with the assistant's
|
||||
reply — a number the agent wrote is not one the operator named.
|
||||
"""
|
||||
text = _FENCE_RE.sub("\n", prompt or "")
|
||||
found: list[int] = []
|
||||
last_end, last_ok = -1, False
|
||||
for m in _NAMED_RE.finditer(text):
|
||||
hashed, digits = bool(m.group(1)), m.group(2)
|
||||
before, after = text[:m.start()], text[m.end():]
|
||||
# A list inherits its head's reading, either way: "PR #198 and #199"
|
||||
# is two PRs, "with 4448, 4449" is two records.
|
||||
if last_end >= 0 and _LIST_JOIN_RE.fullmatch(text[last_end:m.start()]):
|
||||
ok = last_ok
|
||||
elif hashed:
|
||||
ok = _prev_word(before) not in _OTHER_SPACE
|
||||
else:
|
||||
ok = (bool(_OPENS_RE.fullmatch(before))
|
||||
or _prev_word(before) in _REF_WORDS)
|
||||
last_end, last_ok = m.end(), ok
|
||||
if not ok or digits[0] == "0" or len(digits) < (2 if hashed else 3):
|
||||
continue
|
||||
if not hashed and _next_word(after) in _UNITS:
|
||||
continue
|
||||
nid = int(digits)
|
||||
if nid not in found:
|
||||
found.append(nid)
|
||||
if len(found) >= limit:
|
||||
break
|
||||
return found
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Recurring task rule validation, date calculation, and scheduled spawning."""
|
||||
import calendar
|
||||
import logging
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note, TaskStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_VALID_UNITS = {"day", "week", "month", "year"}
|
||||
|
||||
|
||||
def validate_recurrence_rule(rule: dict) -> None:
|
||||
"""Raise ValueError if rule is structurally invalid."""
|
||||
if not isinstance(rule, dict):
|
||||
raise ValueError("recurrence_rule must be an object")
|
||||
rule_type = rule.get("type")
|
||||
if rule_type not in ("interval", "calendar"):
|
||||
raise ValueError("recurrence_rule.type must be 'interval' or 'calendar'")
|
||||
unit = rule.get("unit")
|
||||
if unit not in _VALID_UNITS:
|
||||
raise ValueError(f"recurrence_rule.unit must be one of {_VALID_UNITS}")
|
||||
if rule_type == "interval":
|
||||
every = rule.get("every")
|
||||
if not isinstance(every, int) or every < 1:
|
||||
raise ValueError("recurrence_rule.every must be a positive integer")
|
||||
elif rule_type == "calendar":
|
||||
day = rule.get("day_of_month")
|
||||
if not isinstance(day, int) or not 1 <= day <= 31:
|
||||
raise ValueError("recurrence_rule.day_of_month must be an integer between 1 and 31")
|
||||
if unit == "year":
|
||||
month = rule.get("month")
|
||||
if not isinstance(month, int) or not 1 <= month <= 12:
|
||||
raise ValueError("recurrence_rule.month must be an integer between 1 and 12")
|
||||
elif unit != "month":
|
||||
raise ValueError("calendar recurrence only supports unit 'month' or 'year'")
|
||||
|
||||
|
||||
def _add_months(d: date, months: int) -> date:
|
||||
month = d.month - 1 + months
|
||||
year = d.year + month // 12
|
||||
month = month % 12 + 1
|
||||
day = min(d.day, calendar.monthrange(year, month)[1])
|
||||
return d.replace(year=year, month=month, day=day)
|
||||
|
||||
|
||||
def _next_day_of_month(from_date: date, day: int) -> date:
|
||||
"""Return the next occurrence of day-of-month strictly after from_date."""
|
||||
year, month = from_date.year, from_date.month
|
||||
max_day = calendar.monthrange(year, month)[1]
|
||||
clamped = min(day, max_day)
|
||||
if clamped > from_date.day:
|
||||
return from_date.replace(day=clamped)
|
||||
# Advance one month
|
||||
if month == 12:
|
||||
year, month = year + 1, 1
|
||||
else:
|
||||
month += 1
|
||||
return date(year, month, min(day, calendar.monthrange(year, month)[1]))
|
||||
|
||||
|
||||
def _next_annual_date(from_date: date, month: int, day: int) -> date:
|
||||
"""Return the next occurrence of month/day after from_date."""
|
||||
max_day = calendar.monthrange(from_date.year, month)[1]
|
||||
candidate = date(from_date.year, month, min(day, max_day))
|
||||
if candidate > from_date:
|
||||
return candidate
|
||||
return date(
|
||||
from_date.year + 1,
|
||||
month,
|
||||
min(day, calendar.monthrange(from_date.year + 1, month)[1]),
|
||||
)
|
||||
|
||||
|
||||
def calculate_next_due(rule: dict, from_date: date) -> date:
|
||||
"""Return the next due date after from_date according to rule."""
|
||||
rule_type = rule["type"]
|
||||
unit = rule["unit"]
|
||||
if rule_type == "interval":
|
||||
every = rule["every"]
|
||||
if unit == "day":
|
||||
return from_date + timedelta(days=every)
|
||||
if unit == "week":
|
||||
return from_date + timedelta(weeks=every)
|
||||
if unit == "month":
|
||||
return _add_months(from_date, every)
|
||||
if unit == "year":
|
||||
return _add_months(from_date, every * 12)
|
||||
elif rule_type == "calendar":
|
||||
if unit == "month":
|
||||
return _next_day_of_month(from_date, rule["day_of_month"])
|
||||
if unit == "year":
|
||||
return _next_annual_date(from_date, rule["month"], rule["day_of_month"])
|
||||
raise ValueError(f"Unhandled recurrence rule: {rule!r}")
|
||||
|
||||
|
||||
async def spawn_recurring_tasks() -> int:
|
||||
"""Create child tasks for all recurring tasks whose spawn date has arrived.
|
||||
|
||||
Returns the number of tasks spawned.
|
||||
"""
|
||||
from grimoire.services.notes import create_note
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
spawned = 0
|
||||
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Note).where(
|
||||
and_(
|
||||
Note.recurrence_rule.isnot(None),
|
||||
Note.recurrence_next_spawn_at <= now,
|
||||
# Never spawn children off a trashed parent — that would
|
||||
# resurrect work the user explicitly deleted.
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
tasks = result.scalars().all()
|
||||
|
||||
for task in tasks:
|
||||
base_date = task.due_date or now.date()
|
||||
next_due = calculate_next_due(task.recurrence_rule, base_date)
|
||||
try:
|
||||
await create_note(
|
||||
task.user_id,
|
||||
title=task.title,
|
||||
body=task.body or "",
|
||||
status=TaskStatus.todo.value,
|
||||
priority=task.priority or "none",
|
||||
due_date=next_due,
|
||||
tags=list(task.tags or []),
|
||||
project_id=task.project_id,
|
||||
milestone_id=task.milestone_id,
|
||||
recurrence_rule=task.recurrence_rule,
|
||||
)
|
||||
# No explicit embed here: create_note calls embed_note itself, so
|
||||
# the spawn path is not a second copy of the embedding rule.
|
||||
except Exception:
|
||||
logger.exception("Failed to spawn recurring task %d", task.id)
|
||||
continue
|
||||
|
||||
async with async_session() as session:
|
||||
parent = await session.get(Note, task.id)
|
||||
if parent:
|
||||
parent.recurrence_next_spawn_at = None
|
||||
await session.commit()
|
||||
|
||||
spawned += 1
|
||||
logger.info("Spawned recurring task %d → child due %s", task.id, next_due)
|
||||
|
||||
return spawned
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Scheduler for recurring-task spawning.
|
||||
|
||||
Every 15 minutes, creates the next occurrence of any recurring task whose spawn
|
||||
time has arrived — draining `recurrence_next_spawn_at`, which is armed on task
|
||||
completion. Without this job, recurring tasks would never recur.
|
||||
|
||||
One ScheduledJob (services/scheduler.py), like the other *_scheduler modules.
|
||||
(Formerly event_scheduler.py, which also ran event reminders + CalDAV sync;
|
||||
those were removed when the calendar surface was retired.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from grimoire.services.scheduler import ScheduledJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _run_recurrence_spawn() -> None:
|
||||
from grimoire.services.recurrence import spawn_recurring_tasks # noqa: PLC0415
|
||||
await spawn_recurring_tasks()
|
||||
|
||||
|
||||
_JOB = ScheduledJob("recurrence_spawn", _run_recurrence_spawn, label="Recurring-task spawn")
|
||||
|
||||
|
||||
def start_recurrence_scheduler(loop: asyncio.AbstractEventLoop) -> None:
|
||||
# Spawn the next occurrence of due recurring tasks every 15 minutes.
|
||||
# Without this job, recurrence_next_spawn_at is armed on completion but
|
||||
# never drained, so recurring tasks never recur.
|
||||
_JOB.start(loop, IntervalTrigger(minutes=15), describe="recurring-task spawn every 15m")
|
||||
|
||||
|
||||
def stop_recurrence_scheduler() -> None:
|
||||
_JOB.stop()
|
||||
@@ -0,0 +1,362 @@
|
||||
"""The default shapes of a reply, and the moment each one rides.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
A reply is where the person the work is for finds out what happened, and they
|
||||
were not there while it was done. The shapes make a reply readable to them —
|
||||
conclusion first, the shortest reply that carries the answer, where the work
|
||||
stands. A skill arrives only when the agent decides to load it, so in a long
|
||||
session a shape held in a skill fades out of context exactly when replies get
|
||||
longer.
|
||||
|
||||
So the shapes are product content held here, on the server, and delivered by
|
||||
the moments they belong to rather than by the agent remembering to ask:
|
||||
|
||||
- the CORE applies to every reply, so it rides every turn — in full once per
|
||||
session and after a compaction, and as a one-line pointer otherwise;
|
||||
- each SLICE belongs to one kind of reply, and arrives at the moment that
|
||||
comes just before that kind is written: closing a task comes before a
|
||||
completion report, a structured question before an ask, opening a plan
|
||||
before a plan is put up for review.
|
||||
|
||||
The moment already knows which kind of reply is coming, so there is no "pick
|
||||
the type, then load its shape" step for an agent to skip.
|
||||
|
||||
ONE COPY
|
||||
|
||||
This module is the single source. The `reporting-back` skill keeps the
|
||||
long-form reference (a worked example, the reasoning) and points here; the
|
||||
hooks carry timing and transport and say nothing of their own about shape. An
|
||||
operator's adjustments are `preference` records mounted on the same moments,
|
||||
and arrive beside the default — where the two differ, the preference is what
|
||||
the operator asked for.
|
||||
|
||||
HOW TO WRITE ONE
|
||||
|
||||
Domain-neutral: the work may be software, configuration,
|
||||
infrastructure or anything else a person drives through an agent. "Evidence"
|
||||
is what the reader could open to check; "verified" is whatever checking means
|
||||
for that work. The test beside this module refuses software-only vocabulary.
|
||||
|
||||
Short, because it is paid for in every session's context. The core's budget is
|
||||
asserted; a sentence added to it should replace one.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from grimoire.services import moments
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReplyShape:
|
||||
"""One piece of the default reply shape, and when it arrives."""
|
||||
|
||||
key: str
|
||||
"""Stable identifier: what a delivery records and the Settings view keys on."""
|
||||
|
||||
title: str
|
||||
"""What the piece is, as the Settings view names it."""
|
||||
|
||||
moment: str
|
||||
"""The catalog moment this piece rides — where an operator's own
|
||||
preferences for it are mounted too."""
|
||||
|
||||
delivered: str
|
||||
"""When it arrives, said plainly for the Settings view."""
|
||||
|
||||
text: str
|
||||
"""The shape itself, as the agent reads it."""
|
||||
|
||||
reminder: str
|
||||
"""One line standing in for `text` once the session has been shown it:
|
||||
the pointer form, which also serves as the reminder on later turns."""
|
||||
|
||||
|
||||
CORE_KEY = "core"
|
||||
|
||||
# ~550 tokens at four characters a token. The core is paid for in every
|
||||
# session, so this is the ceiling the test holds it to — not a target. The
|
||||
# core sits close to it, so a sentence added has to replace one.
|
||||
CORE_BUDGET_CHARS = 2200
|
||||
# A slice is paid for only at its moment, and once per session; it can afford
|
||||
# more than the core and still has to stay a slice, not a whole skill.
|
||||
# The asks slice is the largest (~1,530) because it carries who decides what in
|
||||
# full: handing a question back is what an ask is tempted to do.
|
||||
SLICE_BUDGET_CHARS = 1600
|
||||
|
||||
|
||||
_CORE = """\
|
||||
Shape the reply around where the work stands, not the order you did things in. \
|
||||
The reader was not there while you worked.
|
||||
|
||||
- **Conclusion first** — the result, the verdict or the question, then what supports it.
|
||||
- **The shortest reply that carries the answer.** Length is work handed back to the \
|
||||
reader. It is earned by a comparison they asked for, options that need laying side by \
|
||||
side, or numbers that are the point.
|
||||
- **One topic per section; bold the few things that matter.**
|
||||
- **Plain words** — the reader's vocabulary, not names you coined while working.
|
||||
- **Place the work** — the task or plan it belongs to, by id and title, read from the \
|
||||
record rather than recalled. Work with no record behind it says so.
|
||||
- **Answer the standing questions, even with "nothing"**: does anything need them, and \
|
||||
what happens next. A section that explains earns its place only when it changes what \
|
||||
they do or decide; the rest belongs in the record's log.
|
||||
- **Decide what only you can see; ask about direction.** Whether a finding holds, what a \
|
||||
measurement says, whether the work is done: settle it, act, and say why so they can \
|
||||
overrule it. Priorities, choices they will live with, and anything hard to undo are theirs.
|
||||
- **A decision they have made is the input to the work.** Act on it; reopen it only \
|
||||
for new evidence, said once.
|
||||
- **Holding an action until they say yes?** Put it near the top, under "Approval requested".
|
||||
- **End with the one thing they need to decide or do, in bold** — or say there is none.
|
||||
|
||||
By kind:
|
||||
- **Answer** — the answer, then what they could open to check it. An evaluation is \
|
||||
verdict · what exists · the gaps · a recommendation.
|
||||
- **Proposal** — two or three approaches, the trade-off of each, one recommendation. \
|
||||
A review is findings ranked by how much they matter.
|
||||
- **Finding** — what you found, why it happens, and what you decided and did about it.
|
||||
- **Progress** — a line or two. **Blocked** — what stopped, what you tried, what you need.
|
||||
- **Where are we** — the plan and its progress · done · open · needs you · next.
|
||||
|
||||
Before sending, read it as someone who was not there, reading quickly; then once more \
|
||||
for what can go."""
|
||||
|
||||
|
||||
_COMPLETION = """\
|
||||
A completion report, from the `placement` the closing call returned:
|
||||
|
||||
- **Where this sits** — the plan and the step, or the task alone when it has no plan.
|
||||
- **What now works** — outcomes the reader would notice ("you can now…", "X no longer…"), \
|
||||
not the steps behind them; those belong in the task's log.
|
||||
- **How / why** — only the decisions worth knowing, and how it was verified. Anything \
|
||||
that could not be verified is said here rather than left to read as passed.
|
||||
- **Needs you** — an action, an approval, a decision, or "nothing". It has to answer two \
|
||||
questions yes: is it theirs to decide, and is work waiting on it? A question you could settle by \
|
||||
reading or measuring something is work not yet done — settle it, say which way you went, \
|
||||
and leave them free to overrule.
|
||||
- **Next** — from `placement.next`, or say the plan is finished. An offer to fix \
|
||||
something you found goes here."""
|
||||
|
||||
|
||||
_ASKS = """\
|
||||
Asking them to decide or to act — pick the shape:
|
||||
|
||||
- **Decision** — the question first · two to four options, each with what it changes · \
|
||||
your recommendation first among them.
|
||||
- **Clarification** — "My reading is X · the gap is Y · unless you say otherwise I'll do Z."
|
||||
- **Handoff** (only they can do it) — the action · why it needs them · what it unblocks · \
|
||||
what you will do after.
|
||||
- **Approval** (ready, and holding for a yes) — under "Approval requested": exactly what \
|
||||
happens on yes, one numbered item per change so they can approve part · why it needs \
|
||||
them · how it is undone.
|
||||
- **Conflict** (what you are about to do clashes with a rule, a plan or an earlier \
|
||||
decision) — what it says · what you were about to do · where they clash · A or B?
|
||||
|
||||
**Who decides what.** You decide what only you can see the evidence for: whether a \
|
||||
finding holds, what a measurement says, whether a record is right, whether the work is \
|
||||
done, and anything you can settle by reading or measuring. Decide, act, and say why, so \
|
||||
they can overrule it; "your call" on one of these hands them a question they cannot see \
|
||||
into. Evenly balanced evidence is still yours — say which way you went and what would \
|
||||
change your mind. They decide direction: what matters most, what the thing should be, a \
|
||||
trade-off only they can price, anything they will live with afterwards — and every act \
|
||||
that is hard to undo or reaches outside the work.
|
||||
|
||||
When an option rests on existing behaviour, say whose call that was — theirs (cite it) \
|
||||
or a past session's nobody confirmed."""
|
||||
|
||||
|
||||
_PLAN = """\
|
||||
A plan put up for review, before starting:
|
||||
|
||||
- **Goal** — what done looks like, and why, in a sentence or two.
|
||||
- **Steps** — in order, each small enough to check on its own.
|
||||
- **How you will know it works** — something observable, not "it is written".
|
||||
- **Open questions** — only the ones that are theirs to answer."""
|
||||
|
||||
|
||||
SHAPES: dict[str, ReplyShape] = {s.key: s for s in (
|
||||
ReplyShape(CORE_KEY, "Every reply", "reply.report",
|
||||
"every turn — in full once per session and after a compaction, "
|
||||
"otherwise as a one-line pointer", _CORE,
|
||||
"conclusion first; the shortest reply that carries the answer; place the "
|
||||
"work; settle what only you can see and ask about direction; end with the "
|
||||
"one thing they need to do, or say there is none."),
|
||||
ReplyShape("completion", "Completion report", "work.finish",
|
||||
"when a piece of work is closed", _COMPLETION,
|
||||
"where this sits · what now works · how / why, and how it was verified · "
|
||||
"needs you (theirs to decide, and blocking) · next."),
|
||||
ReplyShape("asks", "Asking them to decide or act", "reply.ask",
|
||||
"when a structured question is put to them", _ASKS,
|
||||
"the question first and your recommendation first among the options; "
|
||||
"settle what you can see, and ask only about direction."),
|
||||
ReplyShape("plan", "A plan for review", "work.plan",
|
||||
"when a plan is opened", _PLAN,
|
||||
"goal · steps · how you will know it works · open questions."),
|
||||
)}
|
||||
|
||||
|
||||
def core() -> ReplyShape:
|
||||
return SHAPES[CORE_KEY]
|
||||
|
||||
|
||||
def for_moments(names: list[str]) -> list[ReplyShape]:
|
||||
"""The slices that ride these moments, in catalog order. The core is not a
|
||||
slice: it rides every turn, whatever moment the turn reaches."""
|
||||
wanted = set(names)
|
||||
return [s for s in SHAPES.values() if s.key != CORE_KEY and s.moment in wanted]
|
||||
|
||||
|
||||
def catalog() -> dict:
|
||||
"""The shapes as plain data, for the MCP tool and the REST door alike."""
|
||||
return {
|
||||
"shapes": [
|
||||
{"key": s.key, "title": s.title, "moment": s.moment,
|
||||
"means": moments.MOMENTS[s.moment].means,
|
||||
"delivered": s.delivered, "text": s.text}
|
||||
for s in SHAPES.values()
|
||||
],
|
||||
"total": len(SHAPES),
|
||||
}
|
||||
|
||||
|
||||
# ── Delivery: what a door puts in front of the agent ────────────────────
|
||||
|
||||
# Said once, in the full form: the default is a floor an operator can raise.
|
||||
_FULL_HEAD = ("Reply shape · {title} — the default shape for {what}. Where an operator "
|
||||
"preference shown beside it differs, the preference is what they asked for.")
|
||||
_POINTER = ("Reply shape · {title}, shown in full earlier this session "
|
||||
"(list_reply_shapes has it): {reminder}")
|
||||
|
||||
FULL = "full"
|
||||
POINTER = "pointer"
|
||||
|
||||
|
||||
def render(shape: ReplyShape, *, full: bool) -> str:
|
||||
"""The shape as a door delivers it: in full the first time a session sees
|
||||
it, as its one-line reminder after that."""
|
||||
if not full:
|
||||
return _POINTER.format(title=shape.title, reminder=shape.reminder)
|
||||
what = moments.MOMENTS[shape.moment].means
|
||||
return _FULL_HEAD.format(title=shape.title, what=what) + "\n\n" + shape.text
|
||||
|
||||
|
||||
def deliver(shapes: list[ReplyShape], seen: set[str] | frozenset[str]) -> tuple[list[str], dict[str, str]]:
|
||||
"""Render these shapes against what the session was already shown.
|
||||
|
||||
Returns the blocks and `{key: form}` — every shape delivered and whether
|
||||
it went out in full or as a pointer. The keys sent in full are the ones a
|
||||
door's ledger appends; a door with no ledger passes `seen` empty and
|
||||
every shape goes out in full.
|
||||
"""
|
||||
blocks, forms = [], {}
|
||||
for shape in shapes:
|
||||
full = shape.key not in seen
|
||||
blocks.append(render(shape, full=full))
|
||||
forms[shape.key] = FULL if full else POINTER
|
||||
return blocks, forms
|
||||
|
||||
|
||||
def parse_seen(raw: str | None) -> frozenset[str]:
|
||||
"""A ledger's comma-separated keys, keeping only shapes that exist."""
|
||||
return frozenset(k for k in (p.strip() for p in (raw or "").split(",")) if k in SHAPES)
|
||||
|
||||
|
||||
async def record_delivery(user_id: int | None, forms: dict[str, str], *, via: str) -> None:
|
||||
"""One `reply_shape` row per delivery: which shapes, in which form, by which
|
||||
door. Fire-and-forget — telemetry never takes a shape away from the reader.
|
||||
|
||||
In AppLog beside 409's `report_check`, so the two readings of one reply —
|
||||
was the shape delivered, did the reply carry it — sit in one place.
|
||||
"""
|
||||
if not forms:
|
||||
return
|
||||
try:
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
|
||||
async with async_session() as session:
|
||||
session.add(AppLog(
|
||||
category="plugin", user_id=user_id, action="reply_shape",
|
||||
details=json.dumps({"shapes": forms, "via": via}),
|
||||
))
|
||||
await session.commit()
|
||||
except Exception: # noqa: BLE001 - observation never breaks the observed
|
||||
logger.debug("reply shape delivery not recorded", exc_info=True)
|
||||
|
||||
|
||||
# ── The operator's view ─────────────────────────────────────────────────
|
||||
|
||||
# The window the Settings view counts deliveries over — the same 30 days the
|
||||
# moments panel beside it reads, so the two count the same stretch of work.
|
||||
OVERVIEW_DAYS = 30
|
||||
|
||||
|
||||
async def delivery_counts(user_id: int, *, days: int = OVERVIEW_DAYS) -> dict[str, dict[str, int]]:
|
||||
"""How often each shape went out, in each form, over the last `days`.
|
||||
|
||||
Read from the `reply_shape` rows `record_delivery` writes — one per
|
||||
delivery, naming each shape and whether it went whole or as its reminder.
|
||||
A row whose details do not parse is skipped rather than guessed at.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
counts: dict[str, dict[str, int]] = {k: {FULL: 0, POINTER: 0} for k in SHAPES}
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(AppLog.details).where(
|
||||
AppLog.category == "plugin", AppLog.action == "reply_shape",
|
||||
AppLog.user_id == user_id, AppLog.created_at >= since,
|
||||
)
|
||||
)).scalars().all()
|
||||
for raw in rows:
|
||||
try:
|
||||
forms = json.loads(raw or "{}").get("shapes") or {}
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
for key, form in forms.items() if isinstance(forms, dict) else ():
|
||||
if key in counts and form in counts[key]:
|
||||
counts[key][form] += 1
|
||||
return counts
|
||||
|
||||
|
||||
async def overview(user_id: int, *, days: int = OVERVIEW_DAYS) -> dict:
|
||||
"""The Settings view's read: every shape, the operator's preferences that
|
||||
adjust it, and how often it was delivered.
|
||||
|
||||
A preference adjusts a shape by being MOUNTED on the shape's moment — the
|
||||
same lookup that delivers it beside the shape — so what this lists is
|
||||
exactly what arrives with it. Global homes only: a project's own
|
||||
preferences are that project's, and its rules tab lists them.
|
||||
|
||||
The counts are telemetry and fail open to `deliveries_failed` with "—" in
|
||||
the view; the preference lookup is the substance and is allowed to fail
|
||||
the request, so the view shows an error instead of "no preferences".
|
||||
"""
|
||||
from grimoire.services import rulebooks
|
||||
|
||||
data = catalog()
|
||||
try:
|
||||
counts: dict[str, dict[str, int]] | None = await delivery_counts(user_id, days=days)
|
||||
except Exception: # noqa: BLE001 - a missing count is shown as unknown
|
||||
logger.warning("reply shape delivery counts failed", exc_info=True)
|
||||
counts = None
|
||||
for row in data["shapes"]:
|
||||
mounted = await rulebooks.rules_on_moments(user_id, [row["moment"]])
|
||||
row["preferences"] = [
|
||||
{"id": rule.id, "title": rule.title, "statement": rule.statement}
|
||||
for rule, _at in mounted if rule.kind == "preference"
|
||||
]
|
||||
row["deliveries"] = counts.get(row["key"]) if counts is not None else None
|
||||
data["days"] = days
|
||||
data["deliveries_failed"] = counts is None
|
||||
return data
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Repo -> project binding resolution.
|
||||
|
||||
The SessionStart hook sends the working repo's git remote; this module
|
||||
normalizes it to a stable `repo_key` and resolves it to a project id. The key
|
||||
deliberately collapses ssh/https forms of the same remote so the operator binds
|
||||
a repo once regardless of how it's cloned.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.repo_binding import RepoBinding
|
||||
|
||||
|
||||
def normalize_repo_key(raw: str) -> str:
|
||||
"""Reduce a git remote URL to a stable, scheme-agnostic ``host/owner/repo``.
|
||||
|
||||
Collapses the equivalent clone URLs to one key, e.g.::
|
||||
|
||||
git@git.fabledsword.com:bvandeusen/FabledScribe.git
|
||||
https://git.fabledsword.com/bvandeusen/fabledscribe.git
|
||||
ssh://git@git.fabledsword.com:22/bvandeusen/fabledscribe
|
||||
|
||||
all normalize to ``git.fabledsword.com/bvandeusen/fabledscribe``.
|
||||
|
||||
Returns "" for empty/garbage input so callers can treat it as "no repo".
|
||||
"""
|
||||
s = (raw or "").strip()
|
||||
if not s:
|
||||
return ""
|
||||
|
||||
# scp-like syntax: git@host:owner/repo(.git) -> ssh://host/owner/repo
|
||||
scp = re.match(r"^[^/@]+@([^:/]+):(.+)$", s)
|
||||
if scp and "://" not in s:
|
||||
s = f"//{scp.group(1)}/{scp.group(2)}"
|
||||
else:
|
||||
# strip an explicit scheme (https://, http://, ssh://, git://, git+ssh://)
|
||||
s = re.sub(r"^[a-z][a-z0-9+.\-]*://", "//", s, flags=re.IGNORECASE)
|
||||
if not s.startswith("//"):
|
||||
s = "//" + s
|
||||
|
||||
body = s[2:] # drop leading //
|
||||
# strip userinfo (user@ or user:pass@) on the authority
|
||||
body = re.sub(r"^[^/]*@", "", body)
|
||||
# drop an explicit port on the host (host:22/...)
|
||||
body = re.sub(r"^([^/:]+):\d+", r"\1", body)
|
||||
# strip trailing .git and surrounding slashes
|
||||
body = body.strip("/")
|
||||
if body.endswith(".git"):
|
||||
body = body[:-4]
|
||||
return body.lower()
|
||||
|
||||
|
||||
async def resolve_project(user_id: int, raw_repo: str) -> int | None:
|
||||
"""Return the bound project id for a repo remote, or None if unbound."""
|
||||
key = normalize_repo_key(raw_repo)
|
||||
if not key:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
row = await session.execute(
|
||||
select(RepoBinding.project_id).where(
|
||||
RepoBinding.user_id == user_id, RepoBinding.repo_key == key
|
||||
)
|
||||
)
|
||||
return row.scalar_one_or_none()
|
||||
|
||||
|
||||
async def set_binding(
|
||||
user_id: int, raw_repo: str, project_id: int, ref: str | None = None,
|
||||
) -> RepoBinding:
|
||||
"""Create or update the binding for a repo. Idempotent on (user, repo_key).
|
||||
|
||||
``ref`` is the branch the coverage refresh reads for this
|
||||
binding: a name sets it, ``""`` clears it back to the forge's default
|
||||
branch, ``None`` leaves whatever stands (a re-bind that only moves the
|
||||
project keeps the ref it had).
|
||||
"""
|
||||
key = normalize_repo_key(raw_repo)
|
||||
if not key:
|
||||
raise ValueError("repo remote is empty or unparseable")
|
||||
async with async_session() as session:
|
||||
existing = await session.execute(
|
||||
select(RepoBinding).where(
|
||||
RepoBinding.user_id == user_id, RepoBinding.repo_key == key
|
||||
)
|
||||
)
|
||||
binding = existing.scalar_one_or_none()
|
||||
if binding is None:
|
||||
binding = RepoBinding(user_id=user_id, repo_key=key, project_id=project_id)
|
||||
session.add(binding)
|
||||
else:
|
||||
binding.project_id = project_id
|
||||
if ref is not None:
|
||||
binding.ref = ref.strip() or None
|
||||
await session.commit()
|
||||
await session.refresh(binding)
|
||||
return binding
|
||||
|
||||
|
||||
async def list_bindings(user_id: int) -> list[RepoBinding]:
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(RepoBinding)
|
||||
.where(RepoBinding.user_id == user_id)
|
||||
.order_by(RepoBinding.repo_key)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def bindings_for_project(user_id: int, project_id: int) -> list[RepoBinding]:
|
||||
"""Every binding of a project — key AND the ref its ledger follows."""
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(RepoBinding).where(
|
||||
RepoBinding.user_id == user_id,
|
||||
RepoBinding.project_id == project_id,
|
||||
).order_by(RepoBinding.repo_key)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def is_bound(project_id: int, raw_repo: str) -> str:
|
||||
"""The normalized key when ``raw_repo`` is bound to ``project_id`` by
|
||||
anyone, else "". By anyone, not by the caller: a collaborator judging a
|
||||
shared project's shapes holds no binding of their own, and the question
|
||||
is whether the repo belongs to the project, not who bound it."""
|
||||
key = normalize_repo_key(raw_repo or "")
|
||||
if not key or not project_id:
|
||||
return ""
|
||||
async with async_session() as session:
|
||||
found = await session.execute(
|
||||
select(RepoBinding.id).where(
|
||||
RepoBinding.project_id == project_id,
|
||||
RepoBinding.repo_key == key,
|
||||
).limit(1)
|
||||
)
|
||||
return key if found.first() is not None else ""
|
||||
|
||||
|
||||
async def keys_for_project(user_id: int, project_id: int) -> list[str]:
|
||||
"""Every repo key bound to a project — the snippet→forge join.
|
||||
|
||||
Recorded snippet locations carry free-form repo names ("Grimoire"), which
|
||||
can't address a forge API. The project's binding is the identity that can:
|
||||
a snippet reaches its forge repo through the project it belongs to.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(RepoBinding.repo_key).where(
|
||||
RepoBinding.user_id == user_id,
|
||||
RepoBinding.project_id == project_id,
|
||||
)
|
||||
)
|
||||
return [k for (k,) in rows.all()]
|
||||
|
||||
|
||||
async def bindings_for_key(raw_repo: str) -> list[RepoBinding]:
|
||||
"""All bindings (ANY user) for a repo key — the webhook's entry point.
|
||||
|
||||
A push webhook carries no Grimoire caller, only the repository it happened
|
||||
to; the flag it writes is about each record's truth, so every user who
|
||||
bound the repo gets their project's snippets considered — each write still
|
||||
lands as that record's owner.
|
||||
"""
|
||||
key = normalize_repo_key(raw_repo)
|
||||
if not key:
|
||||
return []
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(RepoBinding).where(RepoBinding.repo_key == key)
|
||||
)
|
||||
return list(rows.scalars().all())
|
||||
|
||||
|
||||
async def delete_binding(user_id: int, raw_repo: str) -> bool:
|
||||
"""Remove a repo's binding. Returns True if a row was deleted."""
|
||||
key = normalize_repo_key(raw_repo)
|
||||
if not key:
|
||||
return False
|
||||
async with async_session() as session:
|
||||
row = await session.execute(
|
||||
select(RepoBinding).where(
|
||||
RepoBinding.user_id == user_id, RepoBinding.repo_key == key
|
||||
)
|
||||
)
|
||||
binding = row.scalar_one_or_none()
|
||||
if binding is None:
|
||||
return False
|
||||
await session.delete(binding)
|
||||
await session.commit()
|
||||
return True
|
||||
@@ -0,0 +1,129 @@
|
||||
"""The report-shape check: does a reply that closed a task carry the completion
|
||||
sections, and what is it sent back with when it does not.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
Everything that helps an agent write a readable completion report arrives
|
||||
BEFORE the reply is written. The Stop hook is the one moment the finished reply
|
||||
exists, so a reply that closed a task is checked for the completion sections
|
||||
(where the work sits, what needs the operator, what comes next). Two jobs:
|
||||
|
||||
- RECORDING the outcome, so the rate of `blocked` among checked replies is a
|
||||
number rather than an impression (`category=plugin, action=report_check`).
|
||||
- OWNING THE WORDS the agent is sent back with, so every client's hook says
|
||||
the same thing (plugin/PACKAGING.md: hooks carry timing and transport).
|
||||
|
||||
ONE END-OF-TURN REQUEST. The Stop hook sends the reply once, with the ids of
|
||||
the tasks the turn closed, and the section check runs here beside the reply
|
||||
hold. The patterns match the words that carry the meaning rather than exact
|
||||
headings, so a shape's wording can change without breaking this.
|
||||
|
||||
app_logs rather than a table of its own: one small event with a JSON detail is
|
||||
what that table holds, it already has retention and an admin viewer, and
|
||||
nothing here needs a join.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
from grimoire.services import reply_shapes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OUTCOMES = ("passed", "blocked", "passed_after_rewrite", "missing_after_rewrite")
|
||||
|
||||
# The sections, in the order the reason lists them, each with what counts as
|
||||
# having it. A bare id ("closed #41") does not place the work: naming a record
|
||||
# by id AND title, or a step position, is the shape — the title is what spares
|
||||
# the reader a lookup. "Needs you: nothing" counts; it is an answer.
|
||||
SECTION_PATTERNS: dict[str, re.Pattern] = {
|
||||
"where it sits": re.compile(
|
||||
r'(#[0-9]+|milestone [0-9]+|task [0-9]+)[*_`]*\s*[*_`]*["“]|step [0-9]+ of [0-9]+', re.I),
|
||||
"needs you": re.compile(
|
||||
r"needs? (from )?you|nothing (is )?needed from you|your (call|decision)", re.I),
|
||||
"next": re.compile(r"\bnext\b", re.I),
|
||||
}
|
||||
SECTIONS = tuple(SECTION_PATTERNS)
|
||||
|
||||
|
||||
def missing_sections(reply: str) -> list[str]:
|
||||
return [name for name, pat in SECTION_PATTERNS.items() if not pat.search(reply or "")]
|
||||
|
||||
|
||||
def known_sections(missing: list[str]) -> list[str]:
|
||||
wanted = {m.strip().lower() for m in missing}
|
||||
return [s for s in SECTIONS if s in wanted]
|
||||
|
||||
|
||||
def block_reason(missing: list[str]) -> str:
|
||||
"""What the agent is told when its completion report is sent back.
|
||||
|
||||
Names what is missing and carries the completion shape's one line rather
|
||||
than the whole shape: the shape itself was delivered when the task closed,
|
||||
and `list_reply_shapes` has it in full.
|
||||
"""
|
||||
listed = ", ".join(known_sections(missing)) or "the completion sections"
|
||||
shape = reply_shapes.SHAPES["completion"]
|
||||
return (
|
||||
f"This turn closed a Grimoire task, and the reply that ends it is missing: {listed}. "
|
||||
"The person the work is for reads this reply to find out where it stands. Rewrite it "
|
||||
f"as a completion report — {shape.reminder} Take where it sits from `placement`, the "
|
||||
"task or plan by id and title (`list_reply_shapes` has the shape in full)."
|
||||
)
|
||||
|
||||
|
||||
async def record_report_check(
|
||||
user_id: int | None,
|
||||
outcome: str,
|
||||
*,
|
||||
missing: list[str] | None = None,
|
||||
task_ids: list[int] | None = None,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
if outcome not in OUTCOMES:
|
||||
raise ValueError(f"unknown report-check outcome {outcome!r}")
|
||||
details: dict = {"outcome": outcome, "missing": known_sections(missing or []),
|
||||
"task_ids": list(task_ids or [])}
|
||||
if project_id:
|
||||
details["project_id"] = project_id
|
||||
async with async_session() as session:
|
||||
session.add(AppLog(
|
||||
category="plugin",
|
||||
user_id=user_id,
|
||||
action="report_check",
|
||||
details=json.dumps(details),
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def check_reply(
|
||||
user_id: int, reply: str, *, task_ids: list[int], rewrite: bool,
|
||||
project_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Check a reply that closed tasks, record the outcome, and say whether it is held.
|
||||
|
||||
`rewrite` is the reply written after an earlier hold: it is recorded and
|
||||
never held again, so a hold costs one turn at most.
|
||||
|
||||
Returns {"outcome", "reason"} — `reason` empty unless the reply is held.
|
||||
A hold happens only when its record was written: an outcome the numbers
|
||||
cannot see is not one this check may act on, so a recording failure
|
||||
degrades to no hold rather than to an unmeasured one.
|
||||
"""
|
||||
missing = missing_sections(reply)
|
||||
if rewrite:
|
||||
outcome = "missing_after_rewrite" if missing else "passed_after_rewrite"
|
||||
else:
|
||||
outcome = "blocked" if missing else "passed"
|
||||
try:
|
||||
await record_report_check(user_id, outcome, missing=missing, task_ids=task_ids,
|
||||
project_id=project_id)
|
||||
except Exception: # noqa: BLE001 - an unrecorded check never holds a reply
|
||||
logger.warning("report check not recorded", exc_info=True)
|
||||
return {"outcome": "", "reason": ""}
|
||||
return {"outcome": outcome,
|
||||
"reason": block_reason(missing) if outcome == "blocked" else ""}
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Carrying a tuned floor across an embedding-model change.
|
||||
|
||||
THE PROBLEM
|
||||
|
||||
A floor is a cosine similarity, and a cosine similarity is a distance in one
|
||||
model's geometry. `bge-small-en-v1.5` → `bge-base-en-v1.5` moves every score on
|
||||
the install at once, in no direction anyone can predict per record. The numbers
|
||||
in `settings` survive the swap unchanged and silently stop describing anything:
|
||||
the bar keeps applying, the telemetry keeps filling, and nothing anywhere says
|
||||
the arm is now cutting in a different place.
|
||||
|
||||
Re-deriving every floor by hand is the alternative. This module lets a tuned
|
||||
threshold be inherited by the next model instead of recalibrated from scratch.
|
||||
|
||||
WHAT TRANSFERS, AND WHY IT IS NOT THE NUMBER
|
||||
|
||||
The raw cosine does not transfer. **The percentile it represented does.** A
|
||||
floor's real content is a decision about SELECTIVITY — "admit roughly the top
|
||||
fifth of what this arm sees" — and that decision is about the operator's
|
||||
tolerance for noise on that surface, not about the embedder. It was true before
|
||||
the model changed and is still true after.
|
||||
|
||||
So: measure what fraction of this surface's calls the old floor admitted, using
|
||||
the scores the old model actually produced; re-score the same queries under the
|
||||
new one; and take the value that admits the same fraction. Same decision, new
|
||||
units.
|
||||
|
||||
WHY `best_available_score` IS THE FIGURE ON BOTH SIDES
|
||||
|
||||
Because it is the one number that exists whether or not a call returned
|
||||
anything — the highest score the corpus offered, before the bar was applied.
|
||||
It is what `retrieval_logs` recorded under the old model, and it is
|
||||
what a re-score reproduces under the new one, so the two sides are the same
|
||||
measurement rather than two things that resemble each other.
|
||||
|
||||
It also makes the re-score cheap to get right: `best_available_score` is
|
||||
computed over the whole candidate set BEFORE `limit` and before any
|
||||
`exclude_ids` the arm passed, so neither has to be reproduced here. Only the
|
||||
corpus FILTERS matter, which is why `_RESCORERS` below carries those and
|
||||
nothing else.
|
||||
|
||||
WHAT THIS DOES NOT DO
|
||||
|
||||
It does not fire by itself, and `migrate_floor` will not write anything unless
|
||||
asked twice — `apply=True` on top of having read the dry run. A model change is
|
||||
exactly the moment when every number is uncertain at once, which is the worst
|
||||
possible moment to let a statistic move every dial unattended. The percentile is
|
||||
a starting point on the new scale, in the same sense the shipped defaults are a
|
||||
starting point: better than a stale number, not a substitute for reading what
|
||||
the surface actually turned away.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.embedding import NoteEmbedding, RuleEmbedding
|
||||
from grimoire.models.retrieval_log import RetrievalLog
|
||||
from grimoire.services.embeddings import (
|
||||
calibration_stamp,
|
||||
rows_off_the_live_model,
|
||||
semantic_search_notes,
|
||||
semantic_search_rules,
|
||||
)
|
||||
from grimoire.services.retrieval_surfaces import floor_for, get_surface
|
||||
from grimoire.services.retrieval_tuning import set_dial
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# How many of a surface's recent calls to re-score. Every one is an embedding
|
||||
# plus a full scan, so this is a real cost — but a percentile off twenty calls
|
||||
# is noise, and the arms that matter here log hundreds a week.
|
||||
DEFAULT_SAMPLE = 200
|
||||
|
||||
|
||||
async def _rescore_rules(user_id: int, query: str, project_id: int | None,
|
||||
kind: str | None) -> float | None:
|
||||
rep: dict = {}
|
||||
await semantic_search_rules(
|
||||
user_id, query, limit=1, threshold=0.0, kind=kind,
|
||||
report=rep, project_id=project_id or None,
|
||||
)
|
||||
return rep.get("best_available_score")
|
||||
|
||||
|
||||
async def _rescore_notes(user_id: int, query: str, project_id: int | None,
|
||||
note_type, task_kind) -> float | None:
|
||||
rep: dict = {}
|
||||
await semantic_search_notes(
|
||||
user_id, query, limit=1, threshold=0.0,
|
||||
project_id=project_id or None, note_type=note_type,
|
||||
task_kind=task_kind, scope="browse", report=rep,
|
||||
)
|
||||
return rep.get("best_available_score")
|
||||
|
||||
|
||||
# ONE re-scorer per surface, carrying that arm's corpus filters and nothing
|
||||
# else. These filters are stated a second time here — the arms in
|
||||
# `plugin_context` are where they are first declared — and that duplication is
|
||||
# deliberate rather than overlooked: the alternative is calling the arms
|
||||
# themselves, which build a menu, write telemetry and record records as
|
||||
# surfaced. A migration that logged two hundred fake retrievals would corrupt
|
||||
# the very table the next tuning decision reads.
|
||||
#
|
||||
# `tests/test_retrieval_migration.py` asserts every registry surface has an
|
||||
# entry, so a seventh arm cannot quietly become un-migratable.
|
||||
_RESCORERS = {
|
||||
"auto_inject": lambda u, q, p: _rescore_notes(u, q, p, None, None),
|
||||
"write_path": lambda u, q, p: _rescore_notes(
|
||||
u, q, p, ("snippet", "note"), "issue"
|
||||
),
|
||||
"write_path_rule": lambda u, q, p: _rescore_rules(u, q, p, None),
|
||||
"pre_tool_rule": lambda u, q, p: _rescore_rules(u, q, p, None),
|
||||
"reply_rule": lambda u, q, p: _rescore_rules(u, q, p, None),
|
||||
"prompt_rule": lambda u, q, p: _rescore_rules(u, q, p, None),
|
||||
}
|
||||
|
||||
# The embedding table each surface's re-scorer reads. A migration from a corpus
|
||||
# that is still part-way through a model change re-scores against a blend of
|
||||
# two geometries and writes a floor computed from it, with a confident reason
|
||||
# attached — so the corpus has to be wholly in the live space first.
|
||||
_CORPUS = {
|
||||
"auto_inject": NoteEmbedding,
|
||||
"write_path": NoteEmbedding,
|
||||
"write_path_rule": RuleEmbedding,
|
||||
"pre_tool_rule": RuleEmbedding,
|
||||
"reply_rule": RuleEmbedding,
|
||||
"prompt_rule": RuleEmbedding,
|
||||
}
|
||||
|
||||
|
||||
def _floor_admitting(scores: list[float], fraction: float) -> float:
|
||||
"""The floor that admits `fraction` of `scores`, on this scale.
|
||||
|
||||
Deliberately exact rather than interpolated: with the scores sorted
|
||||
highest-first, the k-th one IS the bar that admits exactly k. An
|
||||
interpolated quantile would return a number no observed call sits on, which
|
||||
is harder to sanity-check against the sample it came from.
|
||||
|
||||
A fraction rounding to zero returns a floor just above the best score seen —
|
||||
an arm that admitted nothing keeps admitting nothing, rather than being
|
||||
quietly reopened by a migration.
|
||||
"""
|
||||
ranked = sorted(scores, reverse=True)
|
||||
k = int(round(fraction * len(ranked)))
|
||||
if k <= 0:
|
||||
return min(1.0, ranked[0] + 1e-6)
|
||||
return ranked[min(k, len(ranked)) - 1]
|
||||
|
||||
|
||||
async def migrate_floor(
|
||||
user_id: int,
|
||||
surface: str,
|
||||
*,
|
||||
sample: int = DEFAULT_SAMPLE,
|
||||
apply: bool = False,
|
||||
) -> dict:
|
||||
"""Recompute one surface's floor on the current model, preserving selectivity.
|
||||
|
||||
Returns the working: how many calls were sampled, what fraction the old
|
||||
floor admitted, and what value admits the same fraction now. Writes nothing
|
||||
unless `apply=True`, and when it does it writes an ordinary tuning event
|
||||
with the arithmetic in its reason — a migrated floor is reviewable and
|
||||
revertible on exactly the same terms as one a reader chose.
|
||||
"""
|
||||
get_surface(surface) # refuses an unknown name
|
||||
rescore = _RESCORERS.get(surface)
|
||||
if rescore is None:
|
||||
raise ValueError(
|
||||
f"no re-scorer for surface {surface!r}. A surface that cannot be "
|
||||
"re-scored cannot be migrated — add it to _RESCORERS beside the "
|
||||
"arm's own corpus filters."
|
||||
)
|
||||
|
||||
off_model = await rows_off_the_live_model(_CORPUS[surface])
|
||||
if off_model:
|
||||
# Refused before sampling anything, so the order of operations —
|
||||
# re-embed, THEN migrate — is enforced rather than remembered.
|
||||
stamp = calibration_stamp()
|
||||
return {
|
||||
"surface": surface, "migrated": False,
|
||||
"why": f"{off_model} embedding row(s) this surface searches are not "
|
||||
f"yet in {stamp['embedding_model']}'s space. Re-scoring now "
|
||||
"would measure a blend of two models; let the startup "
|
||||
"backfill finish re-embedding, then migrate",
|
||||
"rows_off_model": off_model,
|
||||
"calibration": stamp,
|
||||
}
|
||||
|
||||
old_floor = await floor_for(user_id, surface)
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(
|
||||
RetrievalLog.query,
|
||||
RetrievalLog.project_id,
|
||||
RetrievalLog.best_available_score,
|
||||
)
|
||||
.where(
|
||||
RetrievalLog.source == surface,
|
||||
RetrievalLog.user_id == user_id,
|
||||
RetrievalLog.best_available_score.is_not(None),
|
||||
RetrievalLog.query.is_not(None),
|
||||
RetrievalLog.query != "",
|
||||
)
|
||||
.order_by(RetrievalLog.id.desc())
|
||||
.limit(max(1, int(sample)))
|
||||
)).all()
|
||||
|
||||
if not rows:
|
||||
# Not an error. A surface with no logged calls has no evidence of what
|
||||
# its floor was doing, and inventing a migration for it would be the
|
||||
# exact failure this module's docstring warns about.
|
||||
return {
|
||||
"surface": surface, "migrated": False,
|
||||
"why": "no logged calls carry a best_available_score for this "
|
||||
"surface, so there is no old distribution to preserve",
|
||||
"sampled": 0, "old_floor": old_floor,
|
||||
}
|
||||
|
||||
old_scores = [float(r.best_available_score) for r in rows]
|
||||
admitted = sum(1 for s in old_scores if s >= old_floor)
|
||||
fraction = admitted / len(old_scores)
|
||||
|
||||
new_scores: list[float] = []
|
||||
for r in rows:
|
||||
score = await rescore(user_id, r.query, r.project_id)
|
||||
if score is not None:
|
||||
new_scores.append(float(score))
|
||||
|
||||
if not new_scores:
|
||||
# The corpus answered nothing for any sampled query. Almost always an
|
||||
# embedder that has not finished backfilling under the new model —
|
||||
# migrating from it would set every floor off an empty distribution.
|
||||
return {
|
||||
"surface": surface, "migrated": False,
|
||||
"why": "re-scoring returned nothing for any sampled query — the "
|
||||
"corpus is probably not embedded under the current model yet",
|
||||
"sampled": len(rows), "old_floor": old_floor,
|
||||
"old_admit_rate": round(fraction, 4),
|
||||
}
|
||||
|
||||
proposed = round(min(1.0, max(0.0, _floor_admitting(new_scores, fraction))), 4)
|
||||
stamp = calibration_stamp()
|
||||
reason = (
|
||||
f"Migrated across a calibration change, preserving selectivity: the old "
|
||||
f"floor {old_floor} admitted {admitted} of {len(old_scores)} sampled "
|
||||
f"calls ({fraction:.1%}); {proposed} admits the same share of "
|
||||
f"{len(new_scores)} queries re-scored under "
|
||||
f"{stamp['embedding_model']}/shape {stamp['shape_version']}. The "
|
||||
f"percentile is what carried across, not the number — spot-check the "
|
||||
f"arm before trusting it."
|
||||
)
|
||||
|
||||
result = {
|
||||
"surface": surface,
|
||||
"migrated": False,
|
||||
"sampled": len(rows),
|
||||
"rescored": len(new_scores),
|
||||
"old_floor": old_floor,
|
||||
"old_admit_rate": round(fraction, 4),
|
||||
"old_score_range": [round(min(old_scores), 4), round(max(old_scores), 4)],
|
||||
"new_score_range": [round(min(new_scores), 4), round(max(new_scores), 4)],
|
||||
"proposed_floor": proposed,
|
||||
"calibration": stamp,
|
||||
"reason": reason,
|
||||
}
|
||||
if not apply:
|
||||
return result
|
||||
|
||||
result["migrated"] = True
|
||||
result["applied"] = await set_dial(
|
||||
user_id, surface, "floor", proposed, reason=reason, actor="model",
|
||||
)
|
||||
return result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,292 @@
|
||||
"""Every point that retrieves or surfaces a record, declared.
|
||||
|
||||
WHY A DECLARED LIST WHEN THE ROWS ALREADY NAME THEIR SOURCE. Because the rows
|
||||
can only describe arms that FIRED. A surface that emits nothing writes no row,
|
||||
so it is indistinguishable from a surface that does not exist — and the two
|
||||
call for opposite responses. `retrieval_telemetry` could report perfect health
|
||||
for a window in which half the arms never ran, and a "silent surface"
|
||||
finding could only be made by a human who happened to know which arms were
|
||||
supposed to be there.
|
||||
|
||||
This module is that knowledge, written down. It carries the two facts the rows
|
||||
structurally cannot:
|
||||
|
||||
* **asked or unbidden.** `mcp_search` returning nothing is a search with no
|
||||
matches, which is its job. `auto_inject` returning nothing on every call
|
||||
for a week is an arm that has lost the ability to speak. Same row shape,
|
||||
opposite meanings, and only the caller's intent separates them.
|
||||
* **expected to emit, or deliberately quiet with a reason.** A justified
|
||||
silence must not read as a gap. `rest_*` sources fire only when a
|
||||
person opens the web UI; on an install driven entirely through MCP their
|
||||
absence is correct and permanent, and reporting it every window would
|
||||
train the reader to ignore the list that also carries the real ones.
|
||||
|
||||
NOT `retrieval_surfaces.SURFACES`, and the distinction is deliberate rather
|
||||
than duplication. That registry answers "what can be TUNED" — it carries a
|
||||
floor key, a budget key and the measurement stamp behind their defaults, and
|
||||
it deliberately EXCLUDES the reserved slots (`preference_slot`, `reuse_slot`,
|
||||
`lesson_slot`) because a budget of 1 is their feature and exposing it would
|
||||
invite setting it to 0. This registry answers "what can be MEASURED", and the
|
||||
reserved slots belong in it precisely because they are judgeable without being
|
||||
tunable. One is a control panel; the other is an inventory. A test asserts
|
||||
every tunable surface also appears here, so the two cannot drift apart.
|
||||
|
||||
ADDING A RANKED ARM MEANS WRITING ITS SPEC. The
|
||||
ranked rows are read off `retrieval_pipeline.RANKED`, each spec declaring its
|
||||
own (`Declared`). Any OTHER point — a lookup, an asked search, an ambient
|
||||
carrier, a pull — still means adding a row here. `tests/test_retrieval_registry.py`
|
||||
walks the call sites with the ast module and fails on a source it cannot find
|
||||
below — deliberately not a grep, because two of the sources in this file
|
||||
(`wide_net`, `preference_slot`) reach their recorder through a
|
||||
module constant and a grep for `source="` misses both.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from grimoire.services.retrieval_pipeline import (
|
||||
MOMENT_RULE_SOURCE, NOTE_SOURCES, NOTE_SURFACED_SOURCES, RANKED, RULE_ARMS,
|
||||
RULE_SOURCES,
|
||||
)
|
||||
|
||||
# ── How a point is reached ────────────────────────────────────────────────
|
||||
#
|
||||
# UNBIDDEN: fires on its own, against a query the agent did not write — a
|
||||
# prompt, a file being written, a command about to run. It interrupts, so it
|
||||
# must be able to stay quiet, and a stream of empty calls is a real signal.
|
||||
#
|
||||
# ASKED: someone called a search and wants a ranked list. Returning nothing is
|
||||
# an answer, not a failure, so "cannot decline" must never fire on these.
|
||||
#
|
||||
# AMBIENT: records travel with a reply that was going to be sent anyway (a
|
||||
# project handshake, a planning read). Nothing was ranked and nothing was
|
||||
# chosen, so score-shaped warnings do not apply.
|
||||
#
|
||||
# PULL: a reader opened a record. The terminal event of the whole system, and
|
||||
# the numerator of pull-through.
|
||||
UNBIDDEN = "unbidden"
|
||||
ASKED = "asked"
|
||||
AMBIENT = "ambient"
|
||||
PULL = "pull"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Point:
|
||||
"""One place a record reaches a reader."""
|
||||
|
||||
source: str
|
||||
"""The telemetry `source` value. MUST equal the string the call site passes."""
|
||||
|
||||
kind: str
|
||||
"""UNBIDDEN / ASKED / AMBIENT / PULL — see above."""
|
||||
|
||||
what: str
|
||||
"""One line, for an agent reading a warning that names this source."""
|
||||
|
||||
expects_traffic: bool = True
|
||||
"""Whether silence over an active window is worth reporting.
|
||||
|
||||
False means "this can be quiet forever and that is correct" — and then
|
||||
`quiet_because` must say why, so a reader meets a decision rather than a
|
||||
hole.
|
||||
"""
|
||||
|
||||
quiet_because: str = ""
|
||||
|
||||
logs_unconditionally: bool = True
|
||||
"""Whether this arm writes a row even when it returns NOTHING.
|
||||
|
||||
The reason this field is not merely documentation: an arm that logs only
|
||||
its hits has a zero-result count that is structurally 0, and a "cannot
|
||||
decline" warning computed over that would fire on a LOGGING BUG while
|
||||
reporting a ranking problem, sending the reader to move a floor that was
|
||||
never involved. An arm whose logging has
|
||||
not been confirmed unconditional is not eligible for that warning; it is
|
||||
excluded from the check rather than trusted.
|
||||
"""
|
||||
|
||||
|
||||
def _p(source, kind, what, **kw) -> tuple[str, Point]:
|
||||
return source, Point(source=source, kind=kind, what=what, **kw)
|
||||
|
||||
|
||||
POINTS: dict[str, Point] = dict([
|
||||
# ── Unbidden push arms ───────────────────────────────────────────────
|
||||
#
|
||||
# The RANKED ones are read off the pipeline's specs: every arm, and every stage that runs a query of its own and records
|
||||
# under its own name, declares its row where it is defined. What stays
|
||||
# here is what no spec describes — the lookups below, and everything asked,
|
||||
# ambient or pulled.
|
||||
*(_p(spec.source, UNBIDDEN, spec.declared.what,
|
||||
expects_traffic=not spec.declared.quiet_because,
|
||||
quiet_because=spec.declared.quiet_because)
|
||||
for spec in RANKED),
|
||||
# A LOOKUP, not a ranker: a record the operator named by number
|
||||
# in the message, fetched by id. No score and no bar, so it writes no
|
||||
# retrieval_logs row and no score-shaped warning can apply; its rows are
|
||||
# in `note_usage_events`, where pull-through says whether naming a record
|
||||
# is followed by opening it.
|
||||
_p("named_ref", UNBIDDEN,
|
||||
"a record the operator named by number, looked up directly",
|
||||
expects_traffic=False,
|
||||
quiet_because="speaks only when a message names a record by its id; "
|
||||
"a window in which none did is correctly silent here"),
|
||||
# The moment arm. A LOOKUP, not a ranker: a rule mounted on
|
||||
# a moment arrives when that moment happens, with no score, so it writes no
|
||||
# retrieval_logs row and no score-shaped warning can apply. Its rows are in
|
||||
# `rule_usage_events`, each carrying the moment in `detail`.
|
||||
_p("moment_rule", UNBIDDEN,
|
||||
"rules mounted on a moment of work, delivered when that moment happens",
|
||||
expects_traffic=False,
|
||||
quiet_because="speaks only once some rule is mounted on a moment; an "
|
||||
"install where none is mounted is correctly silent here"),
|
||||
# The rulings arm. A LOOKUP, not a ranker: a path falls
|
||||
# under a System's patterns or it does not, so nothing here writes to
|
||||
# retrieval_logs and no score-shaped warning can apply. Its rows are in
|
||||
# `system_usage_events`, read back as `retrieval_telemetry`'s
|
||||
# `system_usage` block.
|
||||
_p("rulings_pre_tool", UNBIDDEN,
|
||||
"an area's rulings, shown because a command touched its files",
|
||||
expects_traffic=False,
|
||||
quiet_because="speaks only once a System has both path_patterns and a "
|
||||
"Rulings section; an install where none does is "
|
||||
"correctly silent here"),
|
||||
_p("rulings_write_path", UNBIDDEN,
|
||||
"an area's rulings, shown because an edit touched its files",
|
||||
expects_traffic=False,
|
||||
quiet_because="same as rulings_pre_tool — no System with patterns "
|
||||
"and rulings, nothing to show"),
|
||||
|
||||
# ── Asked: a caller wanted a ranked list ─────────────────────────────
|
||||
_p("mcp_search", ASKED, "an agent called search"),
|
||||
_p("wide_net", ASKED, "an agent asked what might apply, with no bar"),
|
||||
_p("rest_search", ASKED, "the web UI searched", expects_traffic=False,
|
||||
quiet_because="only fires when a person uses the web UI; an install "
|
||||
"driven entirely through MCP is correctly silent here"),
|
||||
_p("browse_search", ASKED, "the browse listing searched", expects_traffic=False,
|
||||
quiet_because="same as rest_search — a human-only entry point"),
|
||||
|
||||
# ── Ambient: carried along with a reply already being sent ───────────
|
||||
_p("enter_project", AMBIENT, "the project handshake"),
|
||||
_p("process_skill_sync", AMBIENT, "stored Processes synced into skills"),
|
||||
_p("start_planning", AMBIENT, "the rules listed when a plan is opened"),
|
||||
_p("get_task", AMBIENT, "the rules listed beside a task"),
|
||||
_p("get_project", AMBIENT, "the rules listed beside a project"),
|
||||
_p("get_milestone", AMBIENT, "the rules listed beside a milestone"),
|
||||
|
||||
# ── The write-path menu's own arms ───────────────────────────────────
|
||||
#
|
||||
# These are NOT in retrieval_logs. The place arm carries no score and so
|
||||
# has no row there at all; declaring it here is what lets the arm making
|
||||
# the strongest possible claim, "there is already a canonical helper in
|
||||
# this exact file", be measured.
|
||||
_p("write_path_place", AMBIENT, "a snippet already placed in this very file"),
|
||||
_p("write_path_semantic", AMBIENT, "prior art matched by meaning"),
|
||||
_p("write_path_sync", AMBIENT,
|
||||
"records that should be updated alongside this edit"),
|
||||
|
||||
# ── Pull: a reader opened the record ─────────────────────────────────
|
||||
_p("mcp_get_note", PULL, "an agent opened a note"),
|
||||
_p("mcp_get_task", PULL, "an agent opened a task"),
|
||||
_p("mcp_get_snippet", PULL, "an agent opened a snippet"),
|
||||
_p("mcp_get_process", PULL, "an agent opened a process"),
|
||||
_p("mcp_get_lesson", PULL, "an agent opened a lesson"),
|
||||
_p("mcp_get_rule", PULL, "an agent opened a rule"),
|
||||
_p("mcp_get_system", PULL, "an agent opened a System", expects_traffic=False,
|
||||
quiet_because="a System's rulings arrive in full in the rulings line, "
|
||||
"so opening one is the exception rather than the "
|
||||
"measure; a window without it is not a gap"),
|
||||
_p("rest_note", PULL, "a person opened a note", expects_traffic=False,
|
||||
quiet_because="web UI only"),
|
||||
_p("rest_task", PULL, "a person opened a task", expects_traffic=False,
|
||||
quiet_because="web UI only"),
|
||||
_p("rest_snippet", PULL, "a person opened a snippet", expects_traffic=False,
|
||||
quiet_because="web UI only"),
|
||||
_p("rest_lesson", PULL, "a person opened a lesson", expects_traffic=False,
|
||||
quiet_because="web UI only"),
|
||||
_p("rest_rule", PULL, "a person opened a rule", expects_traffic=False,
|
||||
quiet_because="web UI only"),
|
||||
])
|
||||
|
||||
|
||||
# Call sites that pass `source` as a VARIABLE, and what they can pass.
|
||||
#
|
||||
# WHY THESE ARE DECLARED RATHER THAN RESOLVED. The registry test reads the
|
||||
# call sites with `ast`, which settles a literal and a module constant but not
|
||||
# a value that arrives through a parameter or a dict key. Rather than let
|
||||
# those sites go unchecked — the silent half of a guard that appears to cover
|
||||
# everything — they are named here, and the test FAILS if it meets an
|
||||
# unresolved site that is not in this list.
|
||||
#
|
||||
# THE HONEST LIMIT, stated so nobody over-trusts this: the test pins the SITE,
|
||||
# not the values. Adding a fourth arm inside `plugin_context`'s `by_arm` fan
|
||||
# out would not fail here. What catches that is `assert_registered`, which the
|
||||
# recorders call as the row is written.
|
||||
# Keyed by the EXACT string the extractor produces, matched by equality rather
|
||||
# than by prefix. The paths are relative to `src/` and so begin `scribe/`; a
|
||||
# prefix spelled any other way silently matches nothing, and every site sails
|
||||
# through a check that looks like it is running.
|
||||
FAN_OUT_SITES: dict[str, tuple[str, ...]] = {
|
||||
# The write path's two LOOKUPS. Its ranked arm (`write_path_semantic`)
|
||||
# records through the pipeline's site below.
|
||||
"grimoire/services/plugin_context.py::record_surfaced(source=arm)": (
|
||||
"write_path_place", "write_path_sync",
|
||||
),
|
||||
"grimoire/services/system_rulings.py::record_system_surfaced(source=source)": (
|
||||
"rulings_pre_tool", "rulings_write_path",
|
||||
),
|
||||
"grimoire/services/rulebooks.py::record_rule_surfaced(source=source)": (
|
||||
"enter_project", "start_planning", "get_task", "get_project",
|
||||
"get_milestone",
|
||||
),
|
||||
# The one retrieval pipeline: every ranked arm's call row
|
||||
# and surfacing rows — both corpora — are written by these sites, so
|
||||
# `source` arrives as a value. The values are the pipeline's own specs,
|
||||
# read rather than restated, so an arm added there is declared here by
|
||||
# construction.
|
||||
"grimoire/services/retrieval_pipeline.py::record_retrieval(source=source)": (
|
||||
RULE_SOURCES + NOTE_SOURCES
|
||||
),
|
||||
"grimoire/services/retrieval_pipeline.py::record_surfaced(source=source)": (
|
||||
NOTE_SURFACED_SOURCES
|
||||
),
|
||||
"grimoire/services/retrieval_pipeline.py::record_rule_surfaced(source=source)": (
|
||||
tuple(arm.source for arm in RULE_ARMS)
|
||||
),
|
||||
# The reply moment's mounted half: it records under the
|
||||
# moment arm's source, read from the pipeline's constant so the two doors
|
||||
# cannot drift apart on the name.
|
||||
"grimoire/services/moment_delivery.py::record_rule_surfaced(source=<expression>)": (
|
||||
MOMENT_RULE_SOURCE,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_point(source: str) -> Point | None:
|
||||
"""The registered point for a source, or None if it is not declared."""
|
||||
return POINTS.get(source)
|
||||
|
||||
|
||||
def sources_expected_to_emit() -> list[str]:
|
||||
"""Every point whose silence over an ACTIVE window is worth reporting."""
|
||||
return [s for s, p in POINTS.items() if p.expects_traffic]
|
||||
|
||||
|
||||
def is_registered(source: str) -> bool:
|
||||
"""Is this source declared?
|
||||
|
||||
Read by the TELEMETRY READOUT, not by the recorders, and that placement is
|
||||
the design rather than an accident of where it was easy. Validating at
|
||||
write time would put a dict lookup on every retrieval to catch a mistake
|
||||
that is only ever made once, when an arm is added — and it could not
|
||||
raise anyway, because telemetry that breaks its caller is the worse bug
|
||||
(`record_retrieval` is fire-and-forget for exactly that reason). So the
|
||||
check sits where the cost is already paid and the reader is already
|
||||
looking: a source that appears in the rows and not in this file comes back
|
||||
as an `unregistered_source` warning.
|
||||
|
||||
That is what covers FAN_OUT_SITES above. The static test pins those sites
|
||||
but not the values they can pass, so a fourth arm added inside one would
|
||||
slip past it — and then show up here the first time it fires.
|
||||
"""
|
||||
return source in POINTS
|
||||
@@ -0,0 +1,410 @@
|
||||
"""Judging what a retrieval arm offered — the review pass.
|
||||
|
||||
The usage tables record what the reader DID with a line: opened it or not.
|
||||
Because every menu line carries its matched passage, that action does not
|
||||
stand in for relevance — a line can be unrelated, related and enough
|
||||
as shown, related and already in context, or related and ignored, and all four
|
||||
are "not opened". Only someone reading the query beside the line can tell
|
||||
them apart. This module is how that reading gets done and kept.
|
||||
|
||||
THE SHAPE. `retrieval_logs` already stores each call's query (enriched, as the
|
||||
arm searched it) and its ranked candidates. A review takes a sample of logged
|
||||
calls the reviewer has not judged, RE-RUNS each query with the arm's own
|
||||
parameters to a depth past its budget, and hands back every candidate with the
|
||||
passage that matched. The reviewer records a verdict per line, with a reason.
|
||||
|
||||
WHY RE-RUN rather than replay the logged ids. The log holds ids and scores,
|
||||
not passages, and the reviewer needs the passage — it is what the reader saw.
|
||||
And the depth past the budget is the point: the candidates just under the cut
|
||||
are the ones a budget change would add, and the only way to judge them is to
|
||||
look at them. The re-run is against today's corpus; `missing` names any
|
||||
logged id the re-run no longer finds, so drift is visible rather than silent.
|
||||
|
||||
WHAT TODAY'S CORPUS GETS WRONG. A record written AFTER the call could not have
|
||||
been offered to it, yet it outranks everything: the session that made the call
|
||||
goes on to write the decision, the fix, the work log — often quoting the very
|
||||
message. Left in, it reads as the arm's best hit and inflates `on_point`
|
||||
exactly where the budget question is decided. So the re-run drops records
|
||||
created after the call (`postdated` names them), and marks a line
|
||||
`changed_since_call` when the record or one of its work logs was written after
|
||||
it — its passage may be text the reader never saw.
|
||||
|
||||
WHY IT WRITES NO TELEMETRY. A review is not a retrieval surface — nothing is
|
||||
put in front of a working session — so it records no `retrieval_logs` row and
|
||||
no surfacing. A row here would be a call to the arm that the arm never made.
|
||||
|
||||
ATTENDED, NEVER BULK. Every verdict needs a reason. A verdict without one is a
|
||||
threshold with extra steps, which is the thing this exists to replace.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.models.note_usage import PULLED, NoteUsageEvent
|
||||
from grimoire.models.retrieval_judgment import VERDICTS, RetrievalJudgment
|
||||
from grimoire.models.retrieval_log import RetrievalLog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The arms a review can re-run FAITHFULLY. A re-run with different parameters
|
||||
# from the arm's would judge a menu nobody was shown, so a surface joins this
|
||||
# only with its own search spelled out in `_rerun`.
|
||||
REVIEWABLE = ("auto_inject",)
|
||||
|
||||
DEPTH_DEFAULT = 6
|
||||
DEPTH_MAX = 10
|
||||
SAMPLE_MAX = 20
|
||||
# Extra candidates fetched so that dropping post-dated records still leaves
|
||||
# `depth` lines. Records written after a call cluster at the top of its
|
||||
# re-run, so the slack has to cover a session's worth of them.
|
||||
POSTDATED_SLACK = 15
|
||||
# How long after a call a pull still counts as "opened after". The server has
|
||||
# no session identity (NoteUsageEvent), so this is a window, not a join.
|
||||
OPENED_WINDOW = timedelta(hours=1)
|
||||
|
||||
HOW_TO_JUDGE = (
|
||||
"Read each line as the reader did: the query, then the line's name, kind "
|
||||
"and passage. Judge from those — do NOT open the record, which counts as a "
|
||||
"pull and would credit the arm with an open it never earned. Verdicts: "
|
||||
"`on_point` — someone in the query's situation should read this; "
|
||||
"`adjacent` — same area, but it would not change what they do; "
|
||||
"`unrelated` — a near neighbour in words only. Every verdict needs a "
|
||||
"`reason` naming what in the query and the passage decided it. Judge the "
|
||||
"lines past the budget too (`within_budget: false`): they are the "
|
||||
"candidates a larger budget would add. A line marked `changed_since_call` "
|
||||
"was written to after the call: if its passage is plainly the later text "
|
||||
"(dated after the call, or about what the call led to), leave it unjudged "
|
||||
"— the reader never saw it."
|
||||
)
|
||||
|
||||
|
||||
def _check_source(source: str) -> str:
|
||||
if source not in REVIEWABLE:
|
||||
raise ValueError(f"source must be one of {list(REVIEWABLE)}, got {source!r}")
|
||||
return source
|
||||
|
||||
|
||||
def _after(stamp, call) -> bool:
|
||||
return stamp is not None and call is not None and stamp > call
|
||||
|
||||
|
||||
async def _rerun(
|
||||
user_id: int, row: RetrievalLog, depth: int, report: dict | None = None,
|
||||
) -> list[dict]:
|
||||
"""The logged call's query, searched again the way its arm searches.
|
||||
|
||||
`auto_inject`'s parameters, read from its spec (`retrieval_pipeline.
|
||||
AUTO_INJECT`) rather than restated: which kinds, whose records, and the
|
||||
arm's floor as logged. The reserved slots (reuse, lesson) are left out —
|
||||
each logs its own source, and is judged as that source when it becomes
|
||||
reviewable.
|
||||
|
||||
Records created after the call are dropped before ranking — the call could
|
||||
not have been offered them — and named in `report["postdated"]`.
|
||||
"""
|
||||
# Imported here: plugin_context imports retrieval_telemetry, which reads
|
||||
# this module's judged block.
|
||||
from grimoire.services import retrieval_pipeline as rp
|
||||
from grimoire.services.embeddings import document_title, semantic_search_notes
|
||||
from grimoire.services.retrieval_pipeline import _menu_name, _menu_passage, _record_kind
|
||||
|
||||
rep: dict = {}
|
||||
hits = await semantic_search_notes(
|
||||
user_id, row.query or "",
|
||||
limit=depth + POSTDATED_SLACK,
|
||||
threshold=row.threshold if row.threshold is not None else 0.0,
|
||||
project_id=row.project_id or None,
|
||||
**rp.note_search_filters(rp.AUTO_INJECT),
|
||||
report=rep,
|
||||
)
|
||||
postdated = [int(n.id) for _s, n in hits if _after(n.created_at, row.created_at)]
|
||||
if report is not None:
|
||||
report["postdated"] = postdated
|
||||
hits = [h for h in hits if int(h[1].id) not in postdated][:depth]
|
||||
chunks = rep.get("best_chunk") or {}
|
||||
budget = int(row.limit_n or 0)
|
||||
out = []
|
||||
for rank, (score, note) in enumerate(hits, start=1):
|
||||
name = _menu_name(note.title, note.note_type, note.data, note.body)
|
||||
out.append({
|
||||
"rank": rank,
|
||||
"record_id": int(note.id),
|
||||
"score": round(float(score), 4),
|
||||
"within_budget": rank <= budget,
|
||||
# Work logs are folded in by menus_to_review, which has a session.
|
||||
"changed_since_call": _after(note.updated_at, row.created_at),
|
||||
"kind": _record_kind(note),
|
||||
"name": name,
|
||||
"passage": _menu_passage(
|
||||
document_title(note.title, note.note_type, note.data, note.body),
|
||||
(chunks.get(int(note.id)) or {}).get("text"), name,
|
||||
) or None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def _opened_after(user_id: int, row: RetrievalLog, ids: list[int]) -> set[int]:
|
||||
"""Which of `ids` the user's AGENT pulled within OPENED_WINDOW of the call.
|
||||
|
||||
Agent pulls only — the `mcp_` prefix, per NoteUsageEvent's convention —
|
||||
because the question is whether the line moved the session, not whether
|
||||
the operator later browsed to the record.
|
||||
"""
|
||||
if not ids or row.created_at is None:
|
||||
return set()
|
||||
async with async_session() as session:
|
||||
found = await session.execute(
|
||||
select(NoteUsageEvent.note_id).distinct().where(
|
||||
NoteUsageEvent.user_id == user_id,
|
||||
NoteUsageEvent.event == PULLED,
|
||||
NoteUsageEvent.source.startswith("mcp_", autoescape=True),
|
||||
NoteUsageEvent.note_id.in_(ids),
|
||||
NoteUsageEvent.created_at >= row.created_at,
|
||||
NoteUsageEvent.created_at <= row.created_at + OPENED_WINDOW,
|
||||
)
|
||||
)
|
||||
return {int(i) for (i,) in found.all()}
|
||||
|
||||
|
||||
async def _logged_after(row: RetrievalLog, ids: list[int]) -> set[int]:
|
||||
"""Which of `ids` gained a work log after the call. A log lands in its
|
||||
task's embedded document without always touching the task's `updated_at`
|
||||
(a closed task takes no claim stamp), so it is read here."""
|
||||
from grimoire.models.task_log import TaskLog
|
||||
|
||||
if not ids or row.created_at is None:
|
||||
return set()
|
||||
async with async_session() as session:
|
||||
found = await session.execute(
|
||||
select(TaskLog.task_id).distinct().where(
|
||||
TaskLog.task_id.in_(ids), TaskLog.created_at > row.created_at,
|
||||
)
|
||||
)
|
||||
return {int(i) for (i,) in found.all()}
|
||||
|
||||
|
||||
def _eligible(user_id: int, source: str, since):
|
||||
"""Logged calls of `source` that offered something fresh and that this
|
||||
reviewer has not judged a line of."""
|
||||
judged = exists().where(and_(
|
||||
RetrievalJudgment.retrieval_log_id == RetrievalLog.id,
|
||||
RetrievalJudgment.user_id == user_id,
|
||||
))
|
||||
return (
|
||||
RetrievalLog.user_id == user_id,
|
||||
RetrievalLog.source == source,
|
||||
RetrievalLog.created_at >= since,
|
||||
RetrievalLog.result_count > 0,
|
||||
RetrievalLog.query.is_not(None),
|
||||
~judged,
|
||||
)
|
||||
|
||||
|
||||
async def menus_to_review(
|
||||
user_id: int, *, source: str = "auto_inject", n: int = 5, days: int = 14,
|
||||
depth: int = DEPTH_DEFAULT,
|
||||
) -> dict:
|
||||
"""A random sample of unjudged logged calls, each re-run for judging.
|
||||
|
||||
Random, not newest: a review of the latest calls judges one afternoon's
|
||||
work, and the readout would then describe that afternoon.
|
||||
"""
|
||||
_check_source(source)
|
||||
n = max(1, min(int(n), SAMPLE_MAX))
|
||||
depth = max(1, min(int(depth), DEPTH_MAX))
|
||||
since = datetime.now(timezone.utc) - timedelta(days=max(1, int(days)))
|
||||
where = _eligible(user_id, source, since)
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(RetrievalLog).where(*where).order_by(func.random()).limit(n)
|
||||
)).scalars().all()
|
||||
remaining = (await session.execute(
|
||||
select(func.count()).select_from(RetrievalLog).where(*where)
|
||||
)).scalar_one()
|
||||
|
||||
menus = []
|
||||
for row in rows:
|
||||
rep: dict = {}
|
||||
lines = await _rerun(user_id, row, depth, report=rep)
|
||||
ids = [ln["record_id"] for ln in lines]
|
||||
opened = await _opened_after(user_id, row, ids)
|
||||
relogged = await _logged_after(row, ids)
|
||||
logged = [int(it["id"]) for it in (row.result_ids or [])]
|
||||
for ln in lines:
|
||||
ln["opened_after"] = ln["record_id"] in opened
|
||||
ln["changed_since_call"] = ln["changed_since_call"] or ln["record_id"] in relogged
|
||||
# Shown fresh then. Inside the budget and not logged means a
|
||||
# repeat the arm withheld, or a ranking the corpus has since moved.
|
||||
ln["logged"] = ln["record_id"] in logged
|
||||
rerun_ids = set(ids)
|
||||
menus.append({
|
||||
"log_id": int(row.id),
|
||||
"created_at": iso(row.created_at),
|
||||
"query": row.query,
|
||||
"budget": row.limit_n,
|
||||
"threshold": row.threshold,
|
||||
"logged_ids": logged,
|
||||
# Logged then, not found now: deleted, edited out of reach, or
|
||||
# outranked since. Judge what is here.
|
||||
"missing": [i for i in logged if i not in rerun_ids],
|
||||
# Written after the call, so never offered to it; left out.
|
||||
"postdated": rep.get("postdated", []),
|
||||
"lines": lines,
|
||||
})
|
||||
return {
|
||||
"source": source,
|
||||
"menus": menus,
|
||||
"remaining": int(remaining),
|
||||
"verdicts": list(VERDICTS),
|
||||
"how_to_judge": HOW_TO_JUDGE,
|
||||
}
|
||||
|
||||
|
||||
async def judge_menu(
|
||||
user_id: int, *, log_id: int, verdicts: list[dict],
|
||||
) -> dict:
|
||||
"""Record a verdict, with its reason, on lines of one logged call.
|
||||
|
||||
Rank, score and budget side are re-derived from a fresh re-run rather than
|
||||
taken from the caller, so a verdict cannot be filed against a position the
|
||||
record does not hold. Judging a line again replaces the earlier verdict.
|
||||
"""
|
||||
if not verdicts:
|
||||
raise ValueError("pass at least one verdict")
|
||||
parsed: list[tuple[int, str, str]] = []
|
||||
for v in verdicts:
|
||||
rid = int(v.get("record_id") or 0)
|
||||
verdict = str(v.get("verdict") or "").strip().lower()
|
||||
reason = str(v.get("reason") or "").strip()
|
||||
if verdict not in VERDICTS:
|
||||
raise ValueError(f"verdict must be one of {list(VERDICTS)}, got {verdict!r}")
|
||||
if not reason:
|
||||
raise ValueError(
|
||||
f"record {rid}: a verdict needs its reason — what in the query "
|
||||
f"and the passage decided it. Without one it cannot be re-read."
|
||||
)
|
||||
parsed.append((rid, verdict, reason))
|
||||
|
||||
async with async_session() as session:
|
||||
row = await session.get(RetrievalLog, int(log_id))
|
||||
# Another user's call reads exactly like a missing one: its query is theirs.
|
||||
if row is None or row.user_id != user_id:
|
||||
raise ValueError(f"logged call {log_id} not found")
|
||||
_check_source(row.source)
|
||||
|
||||
lines = {ln["record_id"]: ln for ln in await _rerun(user_id, row, DEPTH_MAX)}
|
||||
unknown = [rid for rid, _v, _r in parsed if rid not in lines]
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"records {unknown} are not in this call's re-run to depth "
|
||||
f"{DEPTH_MAX} — judge the lines menus_to_review returned"
|
||||
)
|
||||
opened = await _opened_after(user_id, row, [rid for rid, _v, _r in parsed])
|
||||
|
||||
async with async_session() as session:
|
||||
existing = {
|
||||
int(j.record_id): j
|
||||
for j in (await session.execute(
|
||||
select(RetrievalJudgment).where(
|
||||
RetrievalJudgment.retrieval_log_id == row.id,
|
||||
RetrievalJudgment.user_id == user_id,
|
||||
RetrievalJudgment.record_id.in_([rid for rid, _v, _r in parsed]),
|
||||
)
|
||||
)).scalars().all()
|
||||
}
|
||||
for rid, verdict, reason in parsed:
|
||||
ln = lines[rid]
|
||||
j = existing.get(rid) or RetrievalJudgment(
|
||||
user_id=user_id, retrieval_log_id=int(row.id), record_id=rid,
|
||||
)
|
||||
j.source = row.source
|
||||
j.query = row.query
|
||||
j.rank = ln["rank"]
|
||||
j.score = ln["score"]
|
||||
j.within_budget = ln["within_budget"]
|
||||
j.verdict = verdict
|
||||
j.reason = reason
|
||||
j.opened_after = rid in opened
|
||||
session.add(j)
|
||||
await session.commit()
|
||||
return {
|
||||
"log_id": int(row.id),
|
||||
"recorded": len(parsed),
|
||||
"replaced": len(existing),
|
||||
}
|
||||
|
||||
|
||||
async def judged_block(user_id: int | None, since) -> dict:
|
||||
"""`retrieval_telemetry`'s judged block: verdicts by rank, per source.
|
||||
|
||||
Counts, not rates, beside each other per rank — the sample is small by
|
||||
nature and a percentage of seven reads as more than it is. `on_point_unopened`
|
||||
is the number the open rate could never show: lines that related and were
|
||||
left closed, which is the passage doing its job or the reader missing it.
|
||||
|
||||
Raises on a failed read; the caller guards it.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(
|
||||
RetrievalJudgment.source,
|
||||
RetrievalJudgment.rank,
|
||||
RetrievalJudgment.within_budget,
|
||||
RetrievalJudgment.verdict,
|
||||
RetrievalJudgment.opened_after,
|
||||
func.count().label("n"),
|
||||
)
|
||||
.where(
|
||||
RetrievalJudgment.user_id == user_id,
|
||||
RetrievalJudgment.created_at >= since,
|
||||
)
|
||||
.group_by(
|
||||
RetrievalJudgment.source, RetrievalJudgment.rank,
|
||||
RetrievalJudgment.within_budget, RetrievalJudgment.verdict,
|
||||
RetrievalJudgment.opened_after,
|
||||
)
|
||||
)).all()
|
||||
calls = dict((await session.execute(
|
||||
select(
|
||||
RetrievalJudgment.source,
|
||||
func.count(func.distinct(RetrievalJudgment.retrieval_log_id)),
|
||||
)
|
||||
.where(
|
||||
RetrievalJudgment.user_id == user_id,
|
||||
RetrievalJudgment.created_at >= since,
|
||||
)
|
||||
.group_by(RetrievalJudgment.source)
|
||||
)).all())
|
||||
|
||||
def empty() -> dict:
|
||||
return {v: 0 for v in VERDICTS} | {"opened": 0}
|
||||
|
||||
out: dict = {}
|
||||
for source, rank, within, verdict, opened, n in rows:
|
||||
block = out.setdefault(source, {
|
||||
"judged_calls": int(calls.get(source, 0)),
|
||||
"judged_lines": 0,
|
||||
"within_budget": empty(),
|
||||
"beyond_budget": empty(),
|
||||
"by_rank": {},
|
||||
"on_point_unopened": 0,
|
||||
})
|
||||
n = int(n)
|
||||
block["judged_lines"] += n
|
||||
side = block["within_budget" if within else "beyond_budget"]
|
||||
at = block["by_rank"].setdefault(int(rank), empty())
|
||||
for bucket in (side, at):
|
||||
if verdict in VERDICTS:
|
||||
bucket[verdict] += n
|
||||
if opened:
|
||||
bucket["opened"] += n
|
||||
if verdict == VERDICTS[0] and not opened:
|
||||
block["on_point_unopened"] += n
|
||||
for block in out.values():
|
||||
block["by_rank"] = [
|
||||
{"rank": r, **c} for r, c in sorted(block["by_rank"].items())
|
||||
]
|
||||
return out
|
||||
@@ -0,0 +1,146 @@
|
||||
"""One registry of the retrieval surfaces, and the two numbers each one has.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
Every push arm has the same shape: a settings key, a default, and a limit. A
|
||||
floor is chosen and adjusted by the model that uses it, with the operator able
|
||||
to touch it — and a tuning surface cannot be consistent for the model if every
|
||||
arm spells its own configuration differently. So the arms do not own their
|
||||
numbers; they read them from here, and the tuning tool, the routes and the
|
||||
Settings UI all enumerate THIS table rather than hard-coding special cases.
|
||||
|
||||
WHAT A FLOOR IS, AND WHAT IT IS NOT
|
||||
|
||||
It is not a relevance judgement. Relevance is decided by the reader, which is
|
||||
the only participant that can read a trigger against a situation — and the
|
||||
injected line says so out loud ("read it before deciding it does not apply").
|
||||
|
||||
A floor answers the cheaper question: *is this worth ranking at all*. `k` is
|
||||
what binds, and `k` is a BUDGET — how much of this surface's attention a
|
||||
candidate list may spend. An arm that fires before every Bash call cannot
|
||||
afford what an arm that fires once a turn can.
|
||||
|
||||
WHY THE DEFAULTS BELOW ARE STARTING POINTS AND NOT ANSWERS
|
||||
|
||||
A cosine score is a distance in BAAI/bge-small-en-v1.5's vector space, measured
|
||||
against one corpus. It cannot transfer to an install with different records,
|
||||
and a shipped default cannot be defended from any one install's telemetry.
|
||||
Grimoire ships a starting point and the means to correct it, so the values carry
|
||||
the measurement that motivated them without claiming to be right for anybody
|
||||
else.
|
||||
|
||||
HOW A FLOOR SHOULD ACTUALLY BE MOVED
|
||||
|
||||
By reading the records the floor refused — `retrieval_telemetry(
|
||||
near_miss_samples=N)` returns them by id — and never by the percentile alone.
|
||||
An arm can log dozens of consecutive declines with the refused score a hair
|
||||
under the bar while every percentile says "lower it", and the refused record,
|
||||
once read, turns out to be a false positive that a lower bar would deliver on
|
||||
every matching act. The statistic and the correct action can point in opposite
|
||||
directions, and only opening the record tells them apart.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from grimoire.services.settings import bounded_float, get_setting
|
||||
from grimoire.services.retrieval_pipeline import ( # noqa: F401 - Surface re-exported
|
||||
TUNED_ARMS, Surface,
|
||||
)
|
||||
|
||||
# A budget nobody should be able to set past. Not a tuning value — a guard on
|
||||
# the worst case, so a mistyped setting cannot turn a menu into a wall of text.
|
||||
# Shared by every surface because it bounds the same thing everywhere: how many
|
||||
# lines a single unsolicited injection may occupy.
|
||||
MAX_BUDGET = 10
|
||||
|
||||
|
||||
# THE TABLE IS READ OFF THE SPECS. Each ranked arm in
|
||||
# `retrieval_pipeline` carries its own `Surface` — the floor and budget pair,
|
||||
# its settings keys and defaults, and the prose a tuner reads — so an arm added
|
||||
# there is tunable by construction, and nothing here can drift from it.
|
||||
SURFACES: dict[str, Surface] = {
|
||||
arm.tuning.name: arm.tuning for arm in TUNED_ARMS if arm.tuning is not None
|
||||
}
|
||||
|
||||
# Reserved slots are deliberately absent. `preference_slot`, `reuse_slot` and
|
||||
# `lesson_slot` borrow their parent arm's floor and are hard-limited to one hit
|
||||
# each, because their entire purpose is to guarantee a single line to a kind of
|
||||
# record that keeps losing a general score contest — or, for
|
||||
# `lesson_slot`, to a kind whose loss is total rather than merely a lost
|
||||
# convenience, since a lesson has no act arm to fall back on and nobody browses
|
||||
# lessons looking for one. A budget of "1" is the feature;
|
||||
# exposing it as tunable would invite setting it to 0 and silently removing the
|
||||
# guarantee.
|
||||
#
|
||||
# They are still logged under their own `source`, which is what keeps them
|
||||
# judgeable without being tunable: `best_available_id` names the record each
|
||||
# bar refused, so a slot that never places, or one that places weak hits, shows
|
||||
# up as evidence rather than as an argument.
|
||||
|
||||
|
||||
def surface_names() -> list[str]:
|
||||
"""Every tunable surface, in a stable order for menus and listings."""
|
||||
return list(SURFACES)
|
||||
|
||||
|
||||
def get_surface(name: str) -> Surface:
|
||||
"""Look one up, refusing an unknown name loudly.
|
||||
|
||||
A typo'd surface must not be writable. Settings keys are free-form strings
|
||||
in a generic table, so a tuning call naming `pretool_rule` would otherwise
|
||||
write a key nothing ever reads — a change that appears to succeed, reports a
|
||||
new value, and alters nothing.
|
||||
"""
|
||||
try:
|
||||
return SURFACES[name]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"unknown retrieval surface {name!r}. Tunable surfaces are: "
|
||||
+ ", ".join(surface_names())
|
||||
) from None
|
||||
|
||||
|
||||
def dial_for_key(key: str) -> tuple[str, str] | None:
|
||||
"""Which `(surface, dial)` a settings key belongs to, or None.
|
||||
|
||||
The registry read backwards, and it exists for one caller: the generic
|
||||
`/api/settings` endpoint, which accepts any key at all. Without this, a
|
||||
floor written through that endpoint moves with no event recorded, and the
|
||||
tuning history says nothing happened — a trail with holes in it, which is
|
||||
worse than no trail because it reads as complete.
|
||||
|
||||
Derived rather than listed so a seventh surface is covered the moment it is
|
||||
added here, which is the only way this stays true.
|
||||
"""
|
||||
for surface in SURFACES.values():
|
||||
if key == surface.floor_key:
|
||||
return (surface.name, "floor")
|
||||
if key == surface.budget_key:
|
||||
return (surface.name, "budget")
|
||||
return None
|
||||
|
||||
|
||||
async def floor_for(user_id: int, name: str) -> float:
|
||||
"""This install's current floor for a surface, clamped to [0, 1]."""
|
||||
s = get_surface(name)
|
||||
raw = await get_setting(user_id, s.floor_key, str(s.floor_default))
|
||||
return bounded_float(raw, s.floor_default)
|
||||
|
||||
|
||||
async def budget_for(user_id: int, name: str) -> int:
|
||||
"""This install's current budget for a surface, clamped to [1, MAX_BUDGET].
|
||||
|
||||
The lower clamp is 1, never 0: a surface turned off is turned off by its
|
||||
`enabled` switch, which says so. A budget of zero would be an arm that runs
|
||||
a search, logs a retrieval, and renders nothing — indistinguishable in the
|
||||
telemetry from a bar nothing cleared, which is the exact confusion this
|
||||
milestone exists to remove.
|
||||
"""
|
||||
s = get_surface(name)
|
||||
raw = await get_setting(user_id, s.budget_key, "")
|
||||
if not raw and s.budget_falls_back_to:
|
||||
raw = await get_setting(user_id, s.budget_falls_back_to, "")
|
||||
try:
|
||||
value = int(float(raw)) if raw else s.budget_default
|
||||
except (TypeError, ValueError):
|
||||
value = s.budget_default
|
||||
return min(MAX_BUDGET, max(1, value))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,486 @@
|
||||
"""Moving a retrieval surface's floor or budget, with the argument attached.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
A floor is chosen and adjusted by the model that uses it; the operator can
|
||||
touch it, but the model handles it most of the time. `retrieval_surfaces`
|
||||
makes the arms describe their numbers the same way. This module is the write
|
||||
half: one call that moves one dial on one surface, records what it was, what
|
||||
it became, who moved it and why, and refuses to do any of that without a
|
||||
reason.
|
||||
|
||||
WHY A REASON IS REQUIRED
|
||||
|
||||
Because a statistic and the correct action can point in opposite directions.
|
||||
An arm can log dozens of consecutive declines with the refused record a hair
|
||||
under the bar, and every percentile in the readout says "lower it" — while
|
||||
the refused record, once read, turns out to be a false positive that a lower
|
||||
bar would attach to every matching act.
|
||||
|
||||
What separates them is opening the record. A required `reason` is the cheapest
|
||||
mechanism that makes that step happen: a caller who must write down why has to
|
||||
have looked, and a caller who writes down something wrong has left the operator
|
||||
a sentence to disagree with. A number moved silently leaves nothing.
|
||||
|
||||
WHAT THIS DELIBERATELY DOES NOT DO
|
||||
|
||||
It does not decide anything itself. There is no rule here that reads a
|
||||
percentile and picks a value, and that absence is the design — the non-goal that
|
||||
survived is *statistical* auto-tuning, precisely because the statistic was the
|
||||
thing that was wrong. The judgement stays with the reader; this module only
|
||||
makes the judgement recordable and reversible.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.retrieval_tuning import RetrievalTuningEvent
|
||||
from grimoire.services.embeddings import calibration_stamp
|
||||
from grimoire.services.retrieval_surfaces import (
|
||||
MAX_BUDGET,
|
||||
budget_for,
|
||||
floor_for,
|
||||
get_surface,
|
||||
surface_names,
|
||||
)
|
||||
from grimoire.services.settings import set_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DIALS = ("floor", "budget")
|
||||
|
||||
# The actor for a change nobody made by hand: the shipped default moved
|
||||
# between releases. Beside "model" and "human".
|
||||
RELEASE_ACTOR = "release"
|
||||
|
||||
# Long enough to say what was read and what it showed; short enough that nobody
|
||||
# pastes a telemetry dump in. The number is not a measurement — it is the point
|
||||
# at which "0.66" stops being an acceptable answer to "why".
|
||||
_MIN_REASON_CHARS = 20
|
||||
|
||||
|
||||
def _clean_reason(reason: str) -> str:
|
||||
"""The guardrail, enforced here rather than by the column.
|
||||
|
||||
`reason` is NOT NULL in the schema, which "" satisfies. A required field
|
||||
that accepts an empty string is a formality, and this one is the whole
|
||||
mechanism — see the module docstring.
|
||||
"""
|
||||
text = (reason or "").strip()
|
||||
if len(text) < _MIN_REASON_CHARS:
|
||||
raise ValueError(
|
||||
"reason is required, and has to say what you read. A floor moved "
|
||||
"without a stated basis is a number nobody can review or revert. "
|
||||
"Name the evidence: which surface's telemetry, and what the refused "
|
||||
"records actually were — `retrieval_telemetry(near_miss_samples=N)` "
|
||||
"returns them by id, and reading them is the step that separates a "
|
||||
"real miss from a bar doing its job."
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _calibration(row, s, live: dict) -> dict:
|
||||
"""What space one dial's number was chosen in, and whether that space moved.
|
||||
|
||||
Three sources, and they are not interchangeable:
|
||||
|
||||
- `tuned` — the dial was moved and the event carries its stamp.
|
||||
The only case where the answer is known.
|
||||
- `shipped` — the dial has never been moved, so the number in force is
|
||||
the registry default, and the registry records what THAT
|
||||
was measured against (`Surface.measured_model`).
|
||||
- `unstamped` — the dial was moved before events carried a stamp. It was
|
||||
measured under something; naming it would be inventing a
|
||||
fact, so `stale` is None rather than True or False.
|
||||
"Unknown" and "fine" must not render the same.
|
||||
|
||||
`model_changed` and `shape_changed` are reported apart because
|
||||
they call for different responses: a new embedding model means every number
|
||||
is a distance in a geometry that no longer exists, while a re-cut document
|
||||
shape means the same records now embed different text. A caller that only
|
||||
ever sees `stale: true` cannot tell those apart.
|
||||
"""
|
||||
if row is None:
|
||||
model, shape, source = s.measured_model, s.measured_shape, "shipped"
|
||||
elif row.embedding_model is None and row.shape_version is None:
|
||||
return {
|
||||
"source": "unstamped", "embedding_model": None,
|
||||
"shape_version": None, "model_changed": None,
|
||||
"shape_changed": None, "stale": None,
|
||||
}
|
||||
else:
|
||||
model, shape, source = row.embedding_model, row.shape_version, "tuned"
|
||||
|
||||
model_changed = model != live["embedding_model"]
|
||||
shape_changed = shape != live["shape_version"]
|
||||
return {
|
||||
"source": source,
|
||||
"embedding_model": model,
|
||||
"shape_version": shape,
|
||||
"model_changed": model_changed,
|
||||
"shape_changed": shape_changed,
|
||||
"stale": model_changed or shape_changed,
|
||||
}
|
||||
|
||||
|
||||
async def current_settings(user_id: int) -> list[dict]:
|
||||
"""Every tunable surface with its live pair and its last stated reason.
|
||||
|
||||
The read side of the tuning surface, and shaped for a reader who is about to
|
||||
change something: the value, what the arm asks and over what corpus and how
|
||||
often — because a floor cannot be moved sensibly without those three — and
|
||||
the reason last given, so the next change argues with the last one instead
|
||||
of overwriting it blind.
|
||||
|
||||
It also carries `calibration` per dial: the embedding model and
|
||||
document shape the number in force was measured in, and whether either has
|
||||
moved since. NOTHING AUTO-RETUNES on the strength of it. A stale stamp says
|
||||
a number is a measurement of a space that no longer exists — it does not
|
||||
say what the number should be now, and a statistic is the wrong thing to
|
||||
answer that question (see the module docstring). The stamp
|
||||
is here so a reader knows which floors to go and re-measure.
|
||||
"""
|
||||
live = calibration_stamp()
|
||||
out: list[dict] = []
|
||||
async with async_session() as session:
|
||||
for name in surface_names():
|
||||
s = get_surface(name)
|
||||
# ONE QUERY PER DIAL, not one `limit(len(DIALS))` over both.
|
||||
# "The newest two rows" is not "the newest row of each kind": a
|
||||
# surface whose floor was moved three times and whose budget was
|
||||
# moved once returns two floor rows, and the budget change
|
||||
# disappears — a missing reason in `last_change` and a WRONG
|
||||
# calibration answer: a tuned dial reporting as "still on the
|
||||
# shipped default", the one state a reader would not think to check.
|
||||
last = {}
|
||||
for dial in DIALS:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.surface == name,
|
||||
# THIS USER'S CHANGES *OR* A RELEASE'S.
|
||||
# A release's rows carry no user id because no user
|
||||
# made them, and taking the newest of the two is
|
||||
# what makes the answer true: a dial the operator
|
||||
# has tuned is explained by their change, and an
|
||||
# untouched one by the release that last shipped
|
||||
# its default. Filtering to the user alone would
|
||||
# report "still on the shipped starting point" for
|
||||
# a default that had in fact moved — the one state
|
||||
# a reader would not think to check.
|
||||
or_(
|
||||
RetrievalTuningEvent.user_id == user_id,
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
),
|
||||
RetrievalTuningEvent.dial == dial,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalars().first()
|
||||
if row is not None:
|
||||
last[dial] = row
|
||||
out.append({
|
||||
"surface": name,
|
||||
"floor": await floor_for(user_id, name),
|
||||
"budget": await budget_for(user_id, name),
|
||||
"floor_default": s.floor_default,
|
||||
"budget_default": s.budget_default,
|
||||
"asks": s.asks,
|
||||
"over": s.over,
|
||||
"fires": s.fires,
|
||||
# Absent rather than empty when a surface has never been moved:
|
||||
# "still on the shipped starting point" is a real state and
|
||||
# should not render as a blank reason somebody wrote.
|
||||
"last_change": {
|
||||
dial: last[dial].to_dict() for dial in DIALS if dial in last
|
||||
},
|
||||
# Always present for BOTH dials, unlike `last_change`: an
|
||||
# untouched dial still has a number in force, and that number
|
||||
# was still measured in some space. Reported, never acted on —
|
||||
# see the note below on why nothing auto-retunes.
|
||||
"calibration": {
|
||||
dial: _calibration(last.get(dial), s, live) for dial in DIALS
|
||||
},
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def set_dial(
|
||||
user_id: int,
|
||||
surface: str,
|
||||
dial: str,
|
||||
value: float,
|
||||
*,
|
||||
reason: str,
|
||||
actor: str = "model",
|
||||
) -> dict:
|
||||
"""Move one dial on one surface, recording the change and its argument.
|
||||
|
||||
Returns the applied value alongside the previous one, so a caller can see
|
||||
that a clamp bit rather than assuming the number it sent is the number in
|
||||
force — the failure being avoided is a tool reporting success for a value
|
||||
the registry silently corrected.
|
||||
"""
|
||||
s = get_surface(surface) # refuses an unknown name
|
||||
if dial not in DIALS:
|
||||
raise ValueError(f"dial must be one of {DIALS}, got {dial!r}")
|
||||
if actor not in ("model", "human"):
|
||||
raise ValueError(f"actor must be 'model' or 'human', got {actor!r}")
|
||||
text = _clean_reason(reason)
|
||||
|
||||
if dial == "floor":
|
||||
old = await floor_for(user_id, surface)
|
||||
applied = min(1.0, max(0.0, float(value)))
|
||||
key, stored = s.floor_key, str(applied)
|
||||
else:
|
||||
old = float(await budget_for(user_id, surface))
|
||||
applied = float(min(MAX_BUDGET, max(1, int(float(value)))))
|
||||
key, stored = s.budget_key, str(int(applied))
|
||||
|
||||
# A change to the value it already has is not a change, and must not be
|
||||
# written. The Settings form re-sends every field on every save, so without
|
||||
# this the history fills with rows saying the operator set six dials to the
|
||||
# numbers they were already on — and a history nobody can skim is one
|
||||
# nobody reads, which costs the surface its entire purpose.
|
||||
#
|
||||
# Reported rather than silently skipped, so a caller that expected to move
|
||||
# something learns that it did not.
|
||||
if abs(applied - old) < 1e-9:
|
||||
return {
|
||||
"surface": surface, "dial": dial, "previous": old,
|
||||
"applied": applied, "clamped": abs(applied - float(value)) > 1e-9,
|
||||
"reason": text, "actor": actor, "unchanged": True,
|
||||
}
|
||||
|
||||
await set_setting(user_id, key, stored)
|
||||
async with async_session() as session:
|
||||
# Stamped with the space this number was chosen in. Read at
|
||||
# write time rather than passed in: the caller measuring a floor and
|
||||
# the caller recording it are the same call, so there is no window in
|
||||
# which they could disagree.
|
||||
stamp = calibration_stamp()
|
||||
session.add(RetrievalTuningEvent(
|
||||
user_id=user_id, surface=surface, dial=dial,
|
||||
old_value=old, new_value=applied, actor=actor, reason=text,
|
||||
embedding_model=stamp["embedding_model"],
|
||||
shape_version=stamp["shape_version"],
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
return {
|
||||
"surface": surface,
|
||||
"dial": dial,
|
||||
"previous": old,
|
||||
"applied": applied,
|
||||
# True when the registry corrected what was asked for. Said out loud
|
||||
# because a caller that believes it set 1.4 will read the next
|
||||
# telemetry as evidence about a bar that was never in force.
|
||||
"clamped": abs(applied - float(value)) > 1e-9,
|
||||
"reason": text,
|
||||
"actor": actor,
|
||||
# Always present, both ways round: a caller that has to test for the
|
||||
# key's absence to learn the answer will eventually forget to.
|
||||
"unchanged": False,
|
||||
}
|
||||
|
||||
|
||||
async def record_release_defaults() -> list[dict]:
|
||||
"""Write down that a RELEASE moved a shipped default.
|
||||
|
||||
THE HOLE THIS FILLS. `retrieval_tuning_events` records dial turns — a
|
||||
person or a model choosing a number. It is silent about the other way a
|
||||
floor moves, which is somebody editing `floor_default` in the registry and
|
||||
shipping it. That change is invisible to every consumer of this table, and
|
||||
one of those consumers is `band_hugs_floor`, which compares a percentile
|
||||
against a floor and has no way to notice they come from different regimes.
|
||||
|
||||
A window that straddles a shipped change holds calls made under two bars,
|
||||
and the warning can report a band sitting "-0.0216 above" its floor — a
|
||||
negative distance above something, which is what a two-population
|
||||
comparison looks like when it finally says so out loud.
|
||||
|
||||
`user_id IS NULL`, because no user did this. A release acts on every
|
||||
account that has not overridden the dial, and writing one row per user
|
||||
would both multiply the row and misattribute it. Readers take the newest of
|
||||
(this user's own change, the release's) — a dial the operator has tuned is
|
||||
explained by their change, and an untouched one by the release.
|
||||
|
||||
`actor="release"`, a third value beside "model" and "human". The column is
|
||||
Text with no CHECK precisely so a new kind of actor is not a migration —
|
||||
the model's own comment says so, and this is the case it anticipated.
|
||||
|
||||
THE FIRST SIGHTING IS A BASELINE, NOT A CHANGE, and is written with
|
||||
`old_value=None`. Nothing moved; the row exists so that the NEXT release
|
||||
has something to be different from. That distinction is load-bearing
|
||||
downstream: a warning suspends itself on a genuine move and must not
|
||||
suspend itself on a fresh install's baseline, and it tells the two apart by
|
||||
exactly this null.
|
||||
|
||||
Idempotent, and safe to call on every boot: a default that matches the last
|
||||
recorded one writes nothing.
|
||||
"""
|
||||
written: list[dict] = []
|
||||
stamp = calibration_stamp()
|
||||
async with async_session() as session:
|
||||
for name in surface_names():
|
||||
s = get_surface(name)
|
||||
for dial, shipped in (
|
||||
("floor", float(s.floor_default)),
|
||||
("budget", float(s.budget_default)),
|
||||
):
|
||||
last = (
|
||||
await session.execute(
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.surface == name,
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
RetrievalTuningEvent.actor == RELEASE_ACTOR,
|
||||
RetrievalTuningEvent.dial == dial,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalars().first()
|
||||
if last is not None and abs(float(last.new_value) - shipped) < 1e-9:
|
||||
continue
|
||||
old = None if last is None else float(last.new_value)
|
||||
reason = (
|
||||
f"Baseline: this release ships {name}'s {dial} at {shipped}. "
|
||||
f"Recorded so a later change to the shipped value has a "
|
||||
f"date and a predecessor to be measured against; nothing "
|
||||
f"moved here."
|
||||
if old is None else
|
||||
f"A release moved {name}'s shipped {dial} from {old} to "
|
||||
f"{shipped}. Not a dial turn — the default in the registry "
|
||||
f"changed, so calls logged either side of this date were "
|
||||
f"made under different bars and cannot be pooled."
|
||||
)
|
||||
session.add(RetrievalTuningEvent(
|
||||
user_id=None, surface=name, dial=dial,
|
||||
old_value=old, new_value=shipped,
|
||||
actor=RELEASE_ACTOR, reason=reason,
|
||||
embedding_model=stamp["embedding_model"],
|
||||
shape_version=stamp["shape_version"],
|
||||
))
|
||||
written.append({
|
||||
"surface": name, "dial": dial,
|
||||
"old_value": old, "new_value": shipped,
|
||||
"baseline": old is None,
|
||||
})
|
||||
if written:
|
||||
await session.commit()
|
||||
return written
|
||||
|
||||
|
||||
async def floor_moves_since(since) -> dict[str, str]:
|
||||
"""Surfaces whose floor genuinely MOVED at or after `since`.
|
||||
|
||||
Genuinely: `old_value IS NOT NULL`, so a baseline row — which records a
|
||||
default without changing it — does not read as a change. Returns surface ->
|
||||
ISO timestamp of the newest such move, for a reader deciding whether a
|
||||
window's numbers can be pooled.
|
||||
|
||||
Covers both kinds of move, a release's and this user's, because the
|
||||
question is about the SAMPLE rather than about who is responsible for it:
|
||||
a floor that moved mid-window splits the calls either way round.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(RetrievalTuningEvent)
|
||||
.where(
|
||||
RetrievalTuningEvent.dial == "floor",
|
||||
RetrievalTuningEvent.old_value.isnot(None),
|
||||
RetrievalTuningEvent.created_at >= since,
|
||||
)
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
)
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
out.setdefault(row.surface, row.created_at.isoformat())
|
||||
return out
|
||||
|
||||
|
||||
async def floor_history_gaps(since) -> dict[str, str | None]:
|
||||
"""Surfaces whose floor history does NOT cover a window starting at `since`.
|
||||
|
||||
The companion `floor_moves_since` needs. That function answers "did this
|
||||
arm's floor move inside the window", and an empty answer must not be read
|
||||
as "no, it held steady". Those are the same answer only while the LEDGER
|
||||
covers the window. Before an arm's first ledger row there are no rows, so
|
||||
it would report "no move" for a period nobody recorded — absence of
|
||||
evidence arriving as evidence of absence.
|
||||
|
||||
Every install passes through this. The ledger's first row for an arm is
|
||||
written when that install first records baselines, so for a window longer
|
||||
than the install is old — or than the arm is old, for an arm added
|
||||
later — there is no history to consult, and the band check must not read
|
||||
that silence as a clean bill of health.
|
||||
|
||||
Returns ONLY the arms with a gap, mapped to the ISO timestamp from which
|
||||
their floor IS knowable — or None when the arm has no floor history at all.
|
||||
An arm absent from the result is fully covered and can be judged. Shaped
|
||||
that way so a caller loops over what it is already judging and asks one
|
||||
question per arm, rather than reasoning about the ledger itself.
|
||||
|
||||
Every row counts here, baselines included, which is the one place
|
||||
`floor_moves_since` and this deliberately disagree. A baseline is not a
|
||||
change — so it is not a move — but it IS the ledger observing the arm, and
|
||||
from that moment on silence genuinely means the floor held.
|
||||
"""
|
||||
out: dict[str, str | None] = {}
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
RetrievalTuningEvent.surface,
|
||||
func.min(RetrievalTuningEvent.created_at),
|
||||
)
|
||||
.where(RetrievalTuningEvent.dial == "floor")
|
||||
.group_by(RetrievalTuningEvent.surface)
|
||||
)
|
||||
).all()
|
||||
earliest = {surface: first for surface, first in rows}
|
||||
for name in surface_names():
|
||||
first = earliest.get(name)
|
||||
if first is None:
|
||||
out[name] = None
|
||||
elif first > since:
|
||||
out[name] = first.isoformat()
|
||||
return out
|
||||
|
||||
|
||||
async def tuning_history(
|
||||
user_id: int, *, surface: str | None = None, limit: int = 20
|
||||
) -> list[dict]:
|
||||
"""What has been moved, newest first — the operator's review surface.
|
||||
|
||||
Scoped to one surface when asked, because the question is almost always
|
||||
"why is THIS arm set like this", and an unscoped list buries one surface's
|
||||
two changes under another's twenty.
|
||||
"""
|
||||
if surface is not None:
|
||||
get_surface(surface) # refuse a typo on the read too
|
||||
async with async_session() as session:
|
||||
q = (
|
||||
select(RetrievalTuningEvent)
|
||||
.where(or_(
|
||||
RetrievalTuningEvent.user_id == user_id,
|
||||
# Shipped-default changes, which belong to no account and
|
||||
# would otherwise be missing from the one surface built
|
||||
# for asking why a number is where it is.
|
||||
RetrievalTuningEvent.user_id.is_(None),
|
||||
))
|
||||
.order_by(RetrievalTuningEvent.created_at.desc())
|
||||
.limit(max(1, min(int(limit), 200)))
|
||||
)
|
||||
if surface is not None:
|
||||
q = q.where(RetrievalTuningEvent.surface == surface)
|
||||
rows = (await session.execute(q)).scalars().all()
|
||||
return [r.to_dict() for r in rows]
|
||||
@@ -0,0 +1,724 @@
|
||||
"""Which rules belong on which moments — proposals and judgments.
|
||||
|
||||
A rule written before moments existed is mounted on nothing, and reaches a
|
||||
session only when its words resemble the work. Mounting the corpus is a
|
||||
judgment about each rule — WHEN does it apply? — and nothing in a rule's row
|
||||
says whether that judgment was ever made. This service records it, from two
|
||||
sources that both stop at a proposal:
|
||||
|
||||
- THE PASS. An agent reads the rules nobody has judged (`unjudged_rules`),
|
||||
proposes moments for each, or says none fits (`propose`). A proposal mounts
|
||||
nothing; a person confirms it (`judge`).
|
||||
- THE SIGNAL. A rule a session OPENS shortly after a moment fired is evidence
|
||||
it belongs on that moment. Recorded per distinct session with the evidence
|
||||
model lesson↔rule soft links use (lesson_rules: three situations, a
|
||||
cooldown), and once it crosses the bar the next open asks the reader to
|
||||
offer the mount (`co_occurred`).
|
||||
|
||||
Neither source mounts anything by itself. A suggestion that delivered its rule
|
||||
would manufacture the opens it counts, and the operator's corpus is theirs to
|
||||
mount.
|
||||
|
||||
THE MISFIRE is the same machinery pointed the other way. A session
|
||||
that was handed a mounted rule at a moment where it did not apply says so
|
||||
(`misfired`), with why and the action that reached the moment. Reports count
|
||||
per distinct DAY — the MCP door is stateless and no session id reaches it, and
|
||||
a misfire seen on three days is a pattern where three in one afternoon may be
|
||||
one piece of work. At the bar the pair becomes an UNMOUNT proposal; the
|
||||
operator takes the rule off (reject), or unmaps the action, or keeps it
|
||||
(confirm), which stops the asking. A rule left unopened is never counted:
|
||||
most delivered rules go unopened, and the reply hold forces an open.
|
||||
|
||||
EDITS ARE JUDGMENTS TOO. A person changing a rule's moments directly says
|
||||
which moments it belongs on, so `record_mount_change` (called from the one
|
||||
write path, `rulebooks.set_rule_moments`) confirms what was added and rejects
|
||||
what was removed — a moment taken off a rule must not be proposed straight
|
||||
back by the signal.
|
||||
|
||||
ACL: rules are owner-scoped, and every read and write here goes
|
||||
through `rulebooks._owned_rules_clause` / `_fetch_owned_rule`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import exists, func, not_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.rule_moment_judgment import (
|
||||
CONFIRMED, NO_MOMENT, REJECTED, SUGGESTED, RuleMomentJudgment,
|
||||
)
|
||||
from grimoire.models.rulebook import Rule, rule_moments as rule_moments_t
|
||||
from grimoire.services import lesson_rules
|
||||
from grimoire.services import moments as moments_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# What a caller says to judge one pair — acts, not the states they produce.
|
||||
VERDICTS = {"confirm": CONFIRMED, "reject": REJECTED}
|
||||
|
||||
# Recorded on the rows an edit wrote, so a reader can tell a direct change from
|
||||
# an accepted proposal.
|
||||
_MOUNTED_NOTE = "mounted by an edit"
|
||||
_UNMOUNTED_NOTE = "unmounted by an edit"
|
||||
# On a "no moment fits" answer a later mount overturned.
|
||||
_OVERTURNED_NOTE = "a moment was mounted after all"
|
||||
|
||||
# How many unjudged rules one pass page hands over. Enough to batch the
|
||||
# reading, few enough that every rule is actually read.
|
||||
PASS_PAGE = 25
|
||||
# How long after a moment an open still counts as following it. Long enough
|
||||
# for a session to see a line, decide it matters and open the rule; short
|
||||
# enough that the moment is still what the work is doing.
|
||||
SIGNAL_WINDOW_SECONDS = 180
|
||||
# Moments too ubiquitous to say anything about the rule opened after them:
|
||||
# nearly every open follows a command or a change, so counting them would
|
||||
# propose every rule onto both.
|
||||
SIGNAL_SKIP = frozenset({"work.run", "work.change"})
|
||||
|
||||
# The reasons a misfire proposal carries, newest kept. Enough for the operator
|
||||
# to see whether the reports agree; few enough to read in the panel.
|
||||
MISFIRE_REASONS = 5
|
||||
# Distinct actions counted per pair. A moment is reached by a handful of
|
||||
# actions; past this the counts stop telling the operator which one to unmap.
|
||||
_MISFIRE_ACTIONS_CAP = 10
|
||||
_MISFIRE_WHY_CHARS = 400
|
||||
# On a mounted pair that has no judgment row — mounted before the rows were
|
||||
# kept (migration 0118) — when its first misfire gives it one.
|
||||
_UNRECORDED_MOUNT_NOTE = "mounted before judgments were recorded"
|
||||
|
||||
|
||||
def _clean_moment(name) -> str:
|
||||
"""A catalog moment, normalised; "" stays "" (no moment fits)."""
|
||||
raw = (name or "").strip() if isinstance(name, str) else ""
|
||||
if raw == NO_MOMENT:
|
||||
return NO_MOMENT
|
||||
return moments_svc.require_moment(raw)
|
||||
|
||||
|
||||
def _int(v) -> int | None:
|
||||
try:
|
||||
i = int(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return i if i > 0 else None
|
||||
|
||||
|
||||
async def _owned_rules(session, user_id: int, rule_ids) -> dict[int, Rule]:
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
ids = [i for i in (_int(r) for r in rule_ids or []) if i]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = (await session.execute(
|
||||
select(Rule).where(Rule.id.in_(ids)).where(_owned_rules_clause(user_id))
|
||||
)).scalars().all()
|
||||
return {r.id: r for r in rows}
|
||||
|
||||
|
||||
async def _mounts(session, rule_ids) -> dict[int, set[str]]:
|
||||
if not rule_ids:
|
||||
return {}
|
||||
rows = (await session.execute(
|
||||
select(rule_moments_t.c.rule_id, rule_moments_t.c.moment)
|
||||
.where(rule_moments_t.c.rule_id.in_(list(rule_ids)))
|
||||
)).all()
|
||||
out: dict[int, set[str]] = {}
|
||||
for rid, moment in rows:
|
||||
out.setdefault(rid, set()).add(moment)
|
||||
return out
|
||||
|
||||
|
||||
async def _rows(session, rule_ids) -> dict[tuple[int, str], RuleMomentJudgment]:
|
||||
if not rule_ids:
|
||||
return {}
|
||||
rows = (await session.execute(
|
||||
select(RuleMomentJudgment).where(RuleMomentJudgment.rule_id.in_(list(rule_ids)))
|
||||
)).scalars().all()
|
||||
return {(r.rule_id, r.moment): r for r in rows}
|
||||
|
||||
|
||||
def _brief(rule: Rule) -> dict:
|
||||
return {
|
||||
"id": rule.id,
|
||||
"title": rule.title,
|
||||
"kind": rule.kind or "rule",
|
||||
"statement": rule.statement,
|
||||
"when_to_apply": rule.when_to_apply or "",
|
||||
"home": "project" if rule.project_id else "global",
|
||||
}
|
||||
|
||||
|
||||
# ── The pass ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _unjudged_clause():
|
||||
"""No mount and no judgment row of any kind — a rule nobody has looked at."""
|
||||
mounted = exists().where(rule_moments_t.c.rule_id == Rule.id)
|
||||
judged = exists().where(RuleMomentJudgment.rule_id == Rule.id)
|
||||
return not_(mounted), not_(judged)
|
||||
|
||||
|
||||
async def unjudged_rules(user_id: int, limit: int = PASS_PAGE, offset: int = 0) -> dict:
|
||||
"""The caller's rules with no moment and no judgment, a page at a time.
|
||||
|
||||
A rule leaves this list the moment any answer is recorded for it — a
|
||||
proposal, a mount, or "no moment fits" — so a pass run in pieces, or by
|
||||
two sessions, does not read the same rule twice.
|
||||
"""
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
limit = max(1, min(int(limit or PASS_PAGE), 100))
|
||||
offset = max(0, int(offset or 0))
|
||||
clauses = (_owned_rules_clause(user_id), *_unjudged_clause())
|
||||
async with async_session() as session:
|
||||
total = (await session.execute(
|
||||
select(func.count()).select_from(Rule).where(*clauses)
|
||||
)).scalar_one()
|
||||
rows = (await session.execute(
|
||||
select(Rule).where(*clauses).order_by(Rule.id).limit(limit).offset(offset)
|
||||
)).scalars().all()
|
||||
return {
|
||||
"rules": [_brief(r) for r in rows],
|
||||
"total_unjudged": int(total),
|
||||
"offset": offset,
|
||||
"catalog": moments_svc.catalog(),
|
||||
}
|
||||
|
||||
|
||||
async def propose(user_id: int, proposals: list[dict]) -> dict:
|
||||
"""Record a pass's proposals: moments for a rule, or that none fits.
|
||||
|
||||
Each item is `{rule_id, moments: [...], why}` or `{rule_id, none: "why"}`.
|
||||
A moment proposal lands `suggested` and mounts nothing. "No moment fits"
|
||||
lands as the agent's own answer (confirmed, moment ""): it changes nothing
|
||||
that surfaces, and it is what keeps the rule off the next pass — the
|
||||
signal can still propose a moment for it later, on evidence.
|
||||
|
||||
A pair already mounted or already judged is skipped and said so; a pair
|
||||
already suggested has its reason refreshed. A bad item is refused alone,
|
||||
with why, and the rest are written.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
proposed, no_moment = 0, 0
|
||||
skipped: list[dict] = []
|
||||
refused: list[dict] = []
|
||||
async with async_session() as session:
|
||||
owned = await _owned_rules(session, user_id, [p.get("rule_id") for p in proposals or []
|
||||
if isinstance(p, dict)])
|
||||
mounts = await _mounts(session, list(owned))
|
||||
existing = await _rows(session, list(owned))
|
||||
for item in proposals or []:
|
||||
if not isinstance(item, dict):
|
||||
refused.append({"item": item, "error": "each proposal is an object"})
|
||||
continue
|
||||
rid = _int(item.get("rule_id"))
|
||||
if rid is None or rid not in owned:
|
||||
refused.append({"rule_id": item.get("rule_id"),
|
||||
"error": "not a rule you own"})
|
||||
continue
|
||||
none_why = (item.get("none") or "").strip() if isinstance(item.get("none"), str) else ""
|
||||
names = item.get("moments")
|
||||
if none_why and names:
|
||||
refused.append({"rule_id": rid, "error": (
|
||||
"moments and none are the two answers to \"when does this rule "
|
||||
"apply?\" — give the one that holds")})
|
||||
continue
|
||||
if none_why:
|
||||
if mounts.get(rid):
|
||||
skipped.append({"rule_id": rid, "moment": NO_MOMENT,
|
||||
"why": "already mounted on " + ", ".join(sorted(mounts[rid]))})
|
||||
continue
|
||||
row = existing.get((rid, NO_MOMENT))
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rid, moment=NO_MOMENT, source="pass")
|
||||
session.add(row)
|
||||
existing[(rid, NO_MOMENT)] = row
|
||||
row.state, row.note, row.judged_at = CONFIRMED, none_why, now
|
||||
no_moment += 1
|
||||
continue
|
||||
try:
|
||||
wanted = moments_svc.require_moments(names if names is not None else []) or []
|
||||
except ValueError as exc:
|
||||
refused.append({"rule_id": rid, "error": str(exc)})
|
||||
continue
|
||||
if not wanted:
|
||||
refused.append({"rule_id": rid, "error": (
|
||||
"name at least one moment, or say none fits with none=\"why\"")})
|
||||
continue
|
||||
why = (item.get("why") or "").strip() if isinstance(item.get("why"), str) else ""
|
||||
for moment in wanted:
|
||||
if moment in mounts.get(rid, set()):
|
||||
skipped.append({"rule_id": rid, "moment": moment, "why": "already mounted"})
|
||||
continue
|
||||
row = existing.get((rid, moment))
|
||||
if row is not None and row.state != SUGGESTED:
|
||||
skipped.append({"rule_id": rid, "moment": moment,
|
||||
"why": f"already {row.state}" + (f": {row.note}" if row.note else "")})
|
||||
continue
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rid, moment=moment, state=SUGGESTED, source="pass")
|
||||
session.add(row)
|
||||
existing[(rid, moment)] = row
|
||||
row.note = why or row.note
|
||||
proposed += 1
|
||||
await session.commit()
|
||||
return {"proposed": proposed, "no_moment": no_moment,
|
||||
"skipped": skipped, "refused": refused}
|
||||
|
||||
|
||||
async def pending(user_id: int, rule_id: int | None = None, limit: int = 200) -> dict:
|
||||
"""The proposals waiting on a judgment, grouped by rule.
|
||||
|
||||
Each rule carries what it is mounted on now, so a proposal is read beside
|
||||
the mounts it would add to.
|
||||
"""
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
limit = max(1, min(int(limit or 200), 500))
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(RuleMomentJudgment, Rule)
|
||||
.join(Rule, Rule.id == RuleMomentJudgment.rule_id)
|
||||
.where(RuleMomentJudgment.state == SUGGESTED, _owned_rules_clause(user_id))
|
||||
)
|
||||
if rule_id:
|
||||
stmt = stmt.where(Rule.id == int(rule_id))
|
||||
rows = (await session.execute(
|
||||
stmt.order_by(Rule.id, RuleMomentJudgment.moment).limit(limit)
|
||||
)).all()
|
||||
mounts = await _mounts(session, {r.id for _, r in rows})
|
||||
order = {name: i for i, name in enumerate(moments_svc.MOMENTS)}
|
||||
grouped: dict[int, dict] = {}
|
||||
for judgment, rule in rows:
|
||||
entry = grouped.setdefault(rule.id, {
|
||||
**_brief(rule),
|
||||
"mounted": sorted(mounts.get(rule.id, set()),
|
||||
key=lambda n: (order.get(n, len(order)), n)),
|
||||
"proposals": [],
|
||||
})
|
||||
entry["proposals"].append({
|
||||
"proposal": "mount",
|
||||
"moment": judgment.moment,
|
||||
"source": judgment.source,
|
||||
"why": judgment.note or "",
|
||||
"evidence": lesson_rules.evidence_summary(judgment.evidence),
|
||||
"created_at": judgment.created_at.isoformat() if judgment.created_at else None,
|
||||
})
|
||||
for judgment, rule, mounted in await _misfire_proposals(user_id, rule_id, limit):
|
||||
entry = grouped.setdefault(rule.id, {
|
||||
**_brief(rule),
|
||||
"mounted": sorted(mounted, key=lambda n: (order.get(n, len(order)), n)),
|
||||
"proposals": [],
|
||||
})
|
||||
entry["proposals"].append(_unmount_item(judgment))
|
||||
rules = sorted(grouped.values(), key=lambda r: r["id"])
|
||||
return {"rules": rules, "total": sum(len(r["proposals"]) for r in rules)}
|
||||
|
||||
|
||||
def _misfire_due_clause():
|
||||
"""Proposed by the misfire signal and not yet kept by the operator."""
|
||||
mf = RuleMomentJudgment.misfire
|
||||
return mf["proposed_at"].astext.isnot(None), mf["kept_at"].astext.is_(None)
|
||||
|
||||
|
||||
async def _misfire_proposals(user_id: int, rule_id: int | None, limit: int):
|
||||
"""The unmount proposals: pairs past the misfire bar, unanswered, and
|
||||
still mounted — a pair taken off since has had its answer."""
|
||||
from grimoire.services.rulebooks import _owned_rules_clause
|
||||
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(RuleMomentJudgment, Rule)
|
||||
.join(Rule, Rule.id == RuleMomentJudgment.rule_id)
|
||||
.where(_owned_rules_clause(user_id), *_misfire_due_clause())
|
||||
)
|
||||
if rule_id:
|
||||
stmt = stmt.where(Rule.id == int(rule_id))
|
||||
rows = (await session.execute(
|
||||
stmt.order_by(Rule.id, RuleMomentJudgment.moment).limit(limit)
|
||||
)).all()
|
||||
mounts = await _mounts(session, {r.id for _, r in rows})
|
||||
return [(j, r, mounts.get(r.id, set())) for j, r in rows
|
||||
if j.moment in mounts.get(r.id, set())]
|
||||
|
||||
|
||||
def _unmount_item(judgment: RuleMomentJudgment) -> dict:
|
||||
mf = judgment.misfire or {}
|
||||
reasons = list(mf.get("reasons") or [])
|
||||
return {
|
||||
"proposal": "unmount",
|
||||
"moment": judgment.moment,
|
||||
"source": "misfire",
|
||||
"why": reasons[-1]["why"] if reasons else "",
|
||||
"reasons": reasons,
|
||||
"reached_by": dict(mf.get("reached_by") or {}),
|
||||
"evidence": lesson_rules.evidence_summary(mf),
|
||||
"created_at": mf.get("first_at"),
|
||||
}
|
||||
|
||||
|
||||
async def judge(user_id: int, judgments: list[dict]) -> dict:
|
||||
"""Confirm or reject proposals — or any (rule, moment) pair, proposed or not.
|
||||
|
||||
Confirm MOUNTS the rule on the moment (alongside what it already has) —
|
||||
on a pair already mounted, it KEEPS the mount and answers a misfire
|
||||
proposal; reject records that it does not belong there and, if it was
|
||||
mounted, unmounts it. Both go through `rulebooks.set_rule_moments`, the one write
|
||||
path, which records the judgment with this item's `note`. A moment of ""
|
||||
judges the "no moment fits" answer itself. A bad item is refused alone.
|
||||
"""
|
||||
from grimoire.services.rulebooks import list_rule_moments, set_rule_moments
|
||||
|
||||
done: list[dict] = []
|
||||
refused: list[dict] = []
|
||||
for item in judgments or []:
|
||||
if not isinstance(item, dict):
|
||||
refused.append({"item": item, "error": "each judgment is an object"})
|
||||
continue
|
||||
rid = _int(item.get("rule_id"))
|
||||
state = VERDICTS.get(str(item.get("verdict") or "").strip().lower())
|
||||
if rid is None or state is None:
|
||||
refused.append({"rule_id": item.get("rule_id"), "moment": item.get("moment"),
|
||||
"error": f"needs rule_id and a verdict, one of {sorted(VERDICTS)}"})
|
||||
continue
|
||||
try:
|
||||
moment = _clean_moment(item.get("moment"))
|
||||
except ValueError as exc:
|
||||
refused.append({"rule_id": rid, "error": str(exc)})
|
||||
continue
|
||||
note = (item.get("note") or "").strip() if isinstance(item.get("note"), str) else ""
|
||||
if moment == NO_MOMENT:
|
||||
ok = await _judge_no_moment(user_id, rid, state, note)
|
||||
else:
|
||||
current = (await list_rule_moments([rid])).get(rid, [])
|
||||
after = (current + [moment] if state == CONFIRMED and moment not in current
|
||||
else [m for m in current if m != moment] if state == REJECTED
|
||||
else current)
|
||||
ok = await set_rule_moments(rid, user_id, after, note=note, judged=[moment],
|
||||
verdict=state) is not None
|
||||
if not ok:
|
||||
refused.append({"rule_id": rid, "moment": moment, "error": "not a rule you own"})
|
||||
continue
|
||||
done.append({"rule_id": rid, "moment": moment, "state": state})
|
||||
return {"judged": done, "refused": refused}
|
||||
|
||||
|
||||
async def _judge_no_moment(user_id: int, rule_id: int, state: str, note: str) -> bool:
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
if not await _owned_rules(session, user_id, [rule_id]):
|
||||
return False
|
||||
row = (await _rows(session, [rule_id])).get((rule_id, NO_MOMENT))
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rule_id, moment=NO_MOMENT, source="edit")
|
||||
session.add(row)
|
||||
row.state, row.judged_at = state, now
|
||||
row.note = note or row.note
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def record_mount_change(
|
||||
session, rule_id: int, before, after, *, note: str = "",
|
||||
judged=(), verdict: str | None = None,
|
||||
) -> None:
|
||||
"""Record the judgments a change of mounts makes, in the caller's session.
|
||||
|
||||
Added moments are confirmed, removed ones rejected — a mount taken off
|
||||
must not be proposed straight back. `judged` names moments a `judge`
|
||||
call decided explicitly, so a rejection of a moment that was never
|
||||
mounted is recorded too. A suggestion that was accepted keeps its source,
|
||||
so the record still says where the idea came from. Any mount overturns a
|
||||
"no moment fits" answer.
|
||||
"""
|
||||
before, after = set(before or ()), set(after or ())
|
||||
added = after - before
|
||||
removed = before - after
|
||||
explicit = {m for m in judged or () if m} - added - removed
|
||||
if not (added or removed or explicit):
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = await _rows(session, [rule_id])
|
||||
|
||||
def put(moment: str, state: str, default_note: str) -> None:
|
||||
row = rows.get((rule_id, moment))
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rule_id, moment=moment, source="edit")
|
||||
session.add(row)
|
||||
rows[(rule_id, moment)] = row
|
||||
row.state, row.judged_at = state, now
|
||||
row.note = note or default_note or row.note
|
||||
|
||||
for moment in added:
|
||||
put(moment, CONFIRMED, _MOUNTED_NOTE)
|
||||
# A fresh mount is a fresh judgment: misfires counted against an
|
||||
# earlier mount of the pair are not evidence against this one.
|
||||
rows[(rule_id, moment)].misfire = None
|
||||
for moment in removed:
|
||||
put(moment, REJECTED, _UNMOUNTED_NOTE)
|
||||
for moment in explicit:
|
||||
if verdict in (CONFIRMED, REJECTED):
|
||||
put(moment, verdict, "")
|
||||
if verdict == CONFIRMED and moment in after:
|
||||
_keep(rows[(rule_id, moment)], note, now)
|
||||
if after:
|
||||
none_row = rows.get((rule_id, NO_MOMENT))
|
||||
if none_row is not None and none_row.state == CONFIRMED:
|
||||
none_row.state, none_row.judged_at = REJECTED, now
|
||||
none_row.note = _OVERTURNED_NOTE
|
||||
|
||||
|
||||
def _keep(row: RuleMomentJudgment, note: str, now: datetime) -> None:
|
||||
"""The operator kept a mount the misfire signal proposed taking off. The
|
||||
reports stay; the asking stops."""
|
||||
mf = row.misfire
|
||||
if not isinstance(mf, dict) or not mf.get("proposed_at") or mf.get("kept_at"):
|
||||
return
|
||||
row.misfire = {**mf, "kept_at": now.isoformat(), "kept_note": note or ""}
|
||||
|
||||
|
||||
# ── The signal ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _proposal_line(rule: Rule, moment: str, evidence: dict) -> str:
|
||||
counts = lesson_rules.evidence_summary(evidence)
|
||||
kind = rule.kind or "rule"
|
||||
return (
|
||||
f"> {kind.capitalize()} #{rule.id} \"{rule.title}\" has been opened just after "
|
||||
f"`{moment}` in {counts['situations']} distinct sessions — it may belong on "
|
||||
f"that moment, where it would arrive without being searched for. Offer the "
|
||||
f"operator the mount in one line; on their yes, `judge_rule_moments("
|
||||
f"[{{\"rule_id\": {rule.id}, \"moment\": \"{moment}\", \"verdict\": \"confirm\", "
|
||||
f"\"note\": \"why\"}}])` mounts it; on a no, `\"reject\"` with their why stops "
|
||||
f"the asking."
|
||||
)
|
||||
|
||||
|
||||
def signal_moments(moments) -> list[str]:
|
||||
"""The moments an open may be counted against: catalog or procedure
|
||||
moments, minus the ubiquitous ones, de-duplicated in order."""
|
||||
out: list[str] = []
|
||||
for m in moments or []:
|
||||
name = (m or "").strip().lower() if isinstance(m, str) else ""
|
||||
if name and name not in SIGNAL_SKIP and moments_svc.is_moment(name) and name not in out:
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
async def co_occurred(
|
||||
user_id: int, rule_id: int, moments, *, situation: str,
|
||||
project_id: int | None = None,
|
||||
) -> str:
|
||||
"""Record that a rule was opened shortly after these moments fired, and
|
||||
return a proposal line once a (rule, moment) pair has earned one.
|
||||
|
||||
`situation` is what makes two opens distinct — the session, so a rule
|
||||
opened after every push in one long session counts once. A pair already
|
||||
mounted or judged gathers nothing. One proposal per open at most. Fails
|
||||
open: this rides a hook, and bookkeeping must never break the read.
|
||||
"""
|
||||
try:
|
||||
return await _co_occurred(user_id, rule_id, moments, situation, project_id)
|
||||
except Exception:
|
||||
logger.warning("rule/moment co-occurrence could not be recorded", exc_info=True)
|
||||
return ""
|
||||
|
||||
|
||||
async def _co_occurred(user_id, rule_id, moments, situation, project_id) -> str:
|
||||
wanted = signal_moments(moments)
|
||||
key = lesson_rules.situation_key("s", situation)
|
||||
rid = _int(rule_id)
|
||||
if not wanted or not key or rid is None:
|
||||
return ""
|
||||
now = datetime.now(timezone.utc)
|
||||
proposal = None
|
||||
async with async_session() as session:
|
||||
owned = await _owned_rules(session, user_id, [rid])
|
||||
rule = owned.get(rid)
|
||||
if rule is None:
|
||||
return ""
|
||||
mounted = (await _mounts(session, [rid])).get(rid, set())
|
||||
existing = await _rows(session, [rid])
|
||||
for moment in wanted:
|
||||
if moment in mounted:
|
||||
continue
|
||||
row = existing.get((rid, moment))
|
||||
if row is not None and row.state != SUGGESTED:
|
||||
continue
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rid, moment=moment, state=SUGGESTED, source="signal")
|
||||
session.add(row)
|
||||
ev = lesson_rules.add_evidence(row.evidence, key, project_id, now)
|
||||
if proposal is None and lesson_rules.proposal_due(ev, now):
|
||||
ev["proposed_at"] = now.isoformat()
|
||||
ev["proposed_count"] = int(ev.get("proposed_count") or 0) + 1
|
||||
proposal = (moment, ev)
|
||||
row.evidence = ev
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Two opens created the same pair at once; the other stands.
|
||||
await session.rollback()
|
||||
return ""
|
||||
if proposal is None:
|
||||
return ""
|
||||
return _proposal_line(rule, *proposal)
|
||||
|
||||
|
||||
# The most acts one open is checked against. The hook sends only the window's
|
||||
# acts, so this bounds a misbehaving client rather than an ordinary session.
|
||||
_ACTS_CAP = 30
|
||||
|
||||
|
||||
async def opened_after(
|
||||
user_id: int, rule_id: int, acts, *, session_id: str,
|
||||
project_id: int | None = None,
|
||||
) -> dict:
|
||||
"""A rule was opened; `acts` are the tool calls that came just before it.
|
||||
|
||||
Each act is a PreToolUse event as the plugin recorded it (`tool_name`,
|
||||
`tool_input`), resolved to its moments by the same mappings delivery
|
||||
uses — so a moment counted here is one that fired, on this install, for
|
||||
that call. Returns `moments` (what the window reached, after the skip
|
||||
list) and `context` (a proposal line, or "").
|
||||
"""
|
||||
from grimoire.services.moment_actions import moments_for
|
||||
|
||||
reached: list[str] = []
|
||||
for act in list(acts or [])[:_ACTS_CAP]:
|
||||
if not isinstance(act, dict):
|
||||
continue
|
||||
tool = str(act.get("tool_name") or "").strip()
|
||||
if not tool:
|
||||
continue
|
||||
tool_input = act.get("tool_input")
|
||||
try:
|
||||
hits = await moments_for(user_id, tool, tool_input if isinstance(tool_input, dict) else {})
|
||||
except Exception:
|
||||
logger.debug("acts before an open could not be resolved", exc_info=True)
|
||||
continue
|
||||
for hit in hits:
|
||||
name = hit.get("moment") if isinstance(hit, dict) else None
|
||||
if name and name not in reached:
|
||||
reached.append(name)
|
||||
moments = signal_moments(reached)
|
||||
context = ""
|
||||
if moments and (session_id or "").strip():
|
||||
context = await co_occurred(
|
||||
user_id, rule_id, moments, situation=session_id.strip(), project_id=project_id,
|
||||
)
|
||||
return {"moments": moments, "context": context}
|
||||
|
||||
|
||||
# ── The misfire: a mounted rule that arrived where it does not apply ─────────
|
||||
|
||||
|
||||
def misfire_situation(now: datetime) -> str:
|
||||
"""One situation per UTC day. The MCP door carries no session id; a day
|
||||
is the coarser, honest unit, and the slower of the two to cross the bar."""
|
||||
return lesson_rules.situation_key("d", now.date().isoformat())
|
||||
|
||||
|
||||
def add_misfire(misfire, key: str, project_id: int | None, now: datetime, *,
|
||||
why: str, reached_by: str) -> dict:
|
||||
"""A NEW misfire dict with this report counted. Pure, like
|
||||
`lesson_rules.add_evidence`, whose counting it reuses: the per-day
|
||||
situations and the bar are the same as every other proposal's."""
|
||||
ev = lesson_rules.add_evidence(misfire, key, project_id, now)
|
||||
reasons = list(ev.get("reasons") or [])
|
||||
reasons.append({"why": why[:_MISFIRE_WHY_CHARS], "reached_by": reached_by,
|
||||
"at": now.isoformat()})
|
||||
ev["reasons"] = reasons[-MISFIRE_REASONS:]
|
||||
actions = dict(ev.get("reached_by") or {})
|
||||
if reached_by and (reached_by in actions or len(actions) < _MISFIRE_ACTIONS_CAP):
|
||||
actions[reached_by] = int(actions.get(reached_by) or 0) + 1
|
||||
ev["reached_by"] = actions
|
||||
return ev
|
||||
|
||||
|
||||
def _unmount_line(rule: Rule, moment: str, misfire: dict) -> str:
|
||||
days = len(misfire.get("situations") or [])
|
||||
actions = ", ".join(f"`{a}` ×{n}" for a, n in sorted(
|
||||
(misfire.get("reached_by") or {}).items(), key=lambda kv: -kv[1]))
|
||||
kind = rule.kind or "rule"
|
||||
return (
|
||||
f"> {kind.capitalize()} #{rule.id} \"{rule.title}\" has been reported arriving at "
|
||||
f"`{moment}` where it did not apply, on {days} distinct days"
|
||||
+ (f" (reached by {actions})" if actions else "")
|
||||
+ ". Offer the operator the fix in one line. If the rule does not belong "
|
||||
f"at that moment, `judge_rule_moments([{{\"rule_id\": {rule.id}, \"moment\": "
|
||||
f"\"{moment}\", \"verdict\": \"reject\", \"note\": \"why\"}}])` takes it off. "
|
||||
"If it is the ACTION that is not that moment here, `unmap_action` for "
|
||||
"it instead (`list_moments` shows the mapping). If they keep it, "
|
||||
"`\"confirm\"` with their why stops the asking."
|
||||
)
|
||||
|
||||
|
||||
async def misfired(
|
||||
user_id: int, rule_id: int, moment: str, *, why: str,
|
||||
reached_by: str = "", project_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Record that a mounted rule arrived at `moment` where it did not apply.
|
||||
|
||||
Only a MOUNT can misfire: a rule that arrived by resemblance is a
|
||||
retrieval question, answered by its trigger. `why` is required — a report
|
||||
without its reason cannot tell the operator what to fix. Returns the
|
||||
count so far and, once the pair crosses the bar, `context`: the line
|
||||
asking the reader to offer the unmount. A kept mount still counts its
|
||||
reports and asks nothing.
|
||||
"""
|
||||
try:
|
||||
name = moments_svc.require_moment((moment or "").strip())
|
||||
except ValueError as exc:
|
||||
return {"recorded": False, "error": str(exc)}
|
||||
why = (why or "").strip() if isinstance(why, str) else ""
|
||||
if not why:
|
||||
return {"recorded": False, "error": (
|
||||
"say why it did not apply here — the reason is what tells the "
|
||||
"operator whether the rule or the action is wrong")}
|
||||
rid = _int(rule_id)
|
||||
reached_by = (reached_by or "").strip()[:200] if isinstance(reached_by, str) else ""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
owned = await _owned_rules(session, user_id, [rid] if rid else [])
|
||||
rule = owned.get(rid) if rid else None
|
||||
if rule is None:
|
||||
return {"recorded": False, "error": "not a rule you own"}
|
||||
mounted = (await _mounts(session, [rid])).get(rid, set())
|
||||
if name not in mounted:
|
||||
return {"recorded": False, "error": (
|
||||
f"rule {rid} is not mounted on {name}"
|
||||
+ (f" (it is on {', '.join(sorted(mounted))})" if mounted else "")
|
||||
+ ". A rule that arrived by its words rather than a mount is "
|
||||
"fixed through its trigger — update_rule(when_to_apply=...).")}
|
||||
row = (await _rows(session, [rid])).get((rid, name))
|
||||
if row is None:
|
||||
row = RuleMomentJudgment(rule_id=rid, moment=name, state=CONFIRMED,
|
||||
source="edit", note=_UNRECORDED_MOUNT_NOTE,
|
||||
judged_at=now)
|
||||
session.add(row)
|
||||
mf = add_misfire(row.misfire, misfire_situation(now), project_id, now,
|
||||
why=why, reached_by=reached_by)
|
||||
kept = bool(mf.get("kept_at"))
|
||||
due = not kept and lesson_rules.proposal_due(mf, now)
|
||||
if due:
|
||||
mf["proposed_at"] = now.isoformat()
|
||||
mf["proposed_count"] = int(mf.get("proposed_count") or 0) + 1
|
||||
row.misfire = mf
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return {"recorded": False, "error": "a report for this pair landed at the same moment; try again"}
|
||||
out = {
|
||||
"recorded": True, "rule_id": rid, "moment": name,
|
||||
"days": len(mf.get("situations") or []),
|
||||
"bar": lesson_rules.PROPOSE_SITUATIONS,
|
||||
"context": _unmount_line(rule, name, mf) if due else "",
|
||||
}
|
||||
if kept:
|
||||
out["kept"] = {"at": mf.get("kept_at"), "note": mf.get("kept_note") or ""}
|
||||
return out
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Which rules a caller may be shown — a rule's HOME, stated once.
|
||||
|
||||
A rule lives in a rulebook topic — GLOBAL, it applies wherever its owner
|
||||
works — or on one project, where it applies to that project and nowhere else.
|
||||
Every read that hands rules to a session answers the same question about that
|
||||
home, so it is answered here and nowhere else:
|
||||
|
||||
- `project_id` unset: global rules only. A caller that does not say gets the
|
||||
safe failure, which is surfacing less rather than another project's rules.
|
||||
- `project_id=N`: global rules plus project N's own, and N's only when the
|
||||
caller can read that project (access.can_read_project, so a shared project's
|
||||
rules reach its collaborators too).
|
||||
- `everywhere=True`: every rule the caller owns, in any home — the explicit
|
||||
whole-rulebook question.
|
||||
|
||||
Rule search and the moment lookup both need this answer. Two copies of a
|
||||
scope clause is how one of them starts surfacing another project's rules.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import or_
|
||||
|
||||
from grimoire.services.access import can_read_project
|
||||
|
||||
|
||||
async def rule_home(user_id: int, project_id: int | None = None, *, everywhere: bool = False):
|
||||
"""The WHERE clause for the rules this caller may be shown here.
|
||||
|
||||
Needs the statement joined through `joined_to_homes`, which outer-joins
|
||||
the three tables the clause reads. A rule is in a topic XOR on a project
|
||||
(migration 0059), so it matches exactly one arm of whichever clause
|
||||
applies.
|
||||
"""
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.rulebook import Rule, Rulebook
|
||||
|
||||
global_rule = Rulebook.owner_user_id == user_id
|
||||
if everywhere:
|
||||
return or_(global_rule, Project.user_id == user_id)
|
||||
if project_id and await can_read_project(user_id, project_id):
|
||||
return or_(global_rule, Rule.project_id == project_id)
|
||||
return global_rule
|
||||
|
||||
|
||||
def joined_to_homes(stmt):
|
||||
"""Outer-join a statement over `Rule` to the tables `rule_home` reads."""
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.rulebook import Rule, Rulebook, RulebookTopic
|
||||
|
||||
return (
|
||||
stmt.outerjoin(RulebookTopic, Rule.topic_id == RulebookTopic.id)
|
||||
.outerjoin(Rulebook, RulebookTopic.rulebook_id == Rulebook.id)
|
||||
.outerjoin(Project, Rule.project_id == Project.id)
|
||||
)
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Rule usage telemetry — did a surfaced rule ever get read?
|
||||
|
||||
The sibling of `note_usage`, for rules.
|
||||
|
||||
Event streams, deliberately independent:
|
||||
|
||||
- SURFACED: a surface put this rule in front of the agent — a ranked arm,
|
||||
unbidden, or a bulk delivery (see AMBIENT VS RANKED below).
|
||||
- PULLED: someone then opened it in full (`get_rule`, or the REST detail
|
||||
route).
|
||||
|
||||
WHY RULES NEED THEIR OWN STREAM. `retrieval_logs` records what the ranker
|
||||
scored, never whether the hint was any use, so even an honest clear-rate cannot
|
||||
settle whether a rule arm earns its lines. The ratio these two streams produce
|
||||
is the missing half, and without it any threshold change is a number picked
|
||||
off a histogram. (A clear-rate can also mislead outright: an arm that logs
|
||||
only the calls that FOUND something reports a decline count of zero at any
|
||||
threshold — a statistic that cannot vary, read as a finding about the corpus.)
|
||||
|
||||
Design notes, mirroring `note_usage`:
|
||||
- Writes are fire-and-forget through `background.spawn`, so telemetry never
|
||||
adds latency to — or can break — the surface it observes. This module does
|
||||
NOT carry its own copy of the strong-reference dance; `background` is the
|
||||
one place that gets it right, and another copy is how one of them drifts.
|
||||
- Failures degrade, but never SILENTLY. `report_telemetry_failure` logs at
|
||||
WARNING and drops one AppLog row per process per site. A telemetry
|
||||
subsystem whose failures go to `logger.debug` can run at zero for weeks,
|
||||
indistinguishable from "nobody uses this".
|
||||
- Reads (`usage_for_rules`) are awaited and aggregated in one round-trip for
|
||||
a whole page, never per row.
|
||||
|
||||
AMBIENT VS RANKED. The note twin splits ranked surfacings from ambient ones
|
||||
because `enter_project` and the skill sync put records in front of the agent
|
||||
without choosing them, and counting those as surfacings makes recency read as
|
||||
popularity. Rules have exactly that shape: the SessionStart preload, and every
|
||||
`rules_payload` surface hand over the whole applicable set at once, chosen by
|
||||
nobody.
|
||||
|
||||
Those bulk surfaces emit too, as ambient. Without them the always-on set's
|
||||
token cost would be certain and its usefulness UNFALSIFIABLE — the one surface
|
||||
whose value is actually in question would be the one exempt from the
|
||||
scoreboard that judges every other. The split is a readout-level `case()`, no
|
||||
migration, because `event` and `source` are plain Text with no CHECK
|
||||
constraint. `source` stays granular so a reader can still tell the preload
|
||||
from `enter_project` from a ranked arm.
|
||||
|
||||
WHY THIS NAMES THE RANKED SOURCES AND THE TWIN NAMES THE AMBIENT ONES. A
|
||||
deliberate divergence, on the failure mode rather than on symmetry. Both shapes
|
||||
fail silently when someone adds a surface and forgets the list, so the question
|
||||
is which list changes more often — and here it is the ambient one. Ranked
|
||||
sources are added when somebody builds a ranker, which is rare and deliberate;
|
||||
bulk ones appear whenever a surface hands rules over, which is most of them.
|
||||
Naming the rare, slow-moving half means a newly-added bulk surface defaults to
|
||||
`ambient`, which merely under-counts it, instead of defaulting to `ranked`,
|
||||
which would quietly pad the pull-through denominator with surfacings nobody
|
||||
chose and make the arm look imprecise. Keep the list that must be remembered
|
||||
as short and as slow-moving as possible.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.models.rule_usage import (
|
||||
APPLIED, DEPARTED, OUTCOMES, PULLED, SURFACED, RuleUsageEvent,
|
||||
)
|
||||
from grimoire.services.background import report_telemetry_failure, spawn
|
||||
from grimoire.services.retrieval_pipeline import MOMENT_RULE_SOURCE, RULE_RANKED_SOURCES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The surfaces that CHOSE the rules they showed. Everything else is ambient —
|
||||
# see the module docstring for why the rare half is the half that gets named.
|
||||
#
|
||||
# Membership is the whole definition of the pull-through denominator: a ranked
|
||||
# surfacing is a claim ("this rule may apply to what you are doing") that a pull
|
||||
# can confirm or refute, while an ambient one is a delivery nobody decided on.
|
||||
# The ranked half is read off the retrieval pipeline's specs: every rule arm
|
||||
# — the reply backstop included, which records only
|
||||
# the rule that HELD the reply — the preference slot, which is a ranker's
|
||||
# choice twice over, and the rule reached through a confirmed lesson link,
|
||||
# whose own source lets its pull-through read apart from a direct match.
|
||||
# An arm added there is counted here by construction.
|
||||
RANKED_SOURCES = (
|
||||
*RULE_RANKED_SOURCES,
|
||||
# A rule mounted on a moment. Not a ranker's pick, but
|
||||
# not bulk either: somebody decided this rule applies at this moment, and
|
||||
# that is exactly the claim a pull can confirm or refute. Its
|
||||
# pull-through is the evidence for whether a mount earns its line.
|
||||
MOMENT_RULE_SOURCE,
|
||||
)
|
||||
|
||||
|
||||
def is_ambient(source: str) -> bool:
|
||||
"""Was this surfacing a bulk delivery rather than a ranked choice?
|
||||
|
||||
One definition, read by both the per-rule badge readout and the aggregate
|
||||
in `retrieval_telemetry`, so the two cannot disagree about what
|
||||
"surfaced" counts.
|
||||
|
||||
Sync and pure, per the service canon, but deliberately PUBLIC where
|
||||
that canon says such helpers stay `_private`. The departure is the point:
|
||||
a module-private copy in each caller is exactly the second definition this
|
||||
exists to prevent.
|
||||
"""
|
||||
return source not in RANKED_SOURCES
|
||||
|
||||
|
||||
async def _report_failure(site: str) -> None:
|
||||
await report_telemetry_failure("rule_usage", site)
|
||||
|
||||
|
||||
async def _insert_events(rows: list[dict]) -> None:
|
||||
"""Persist usage rows. Best-effort: failures degrade, visibly."""
|
||||
try:
|
||||
async with async_session() as session:
|
||||
session.add_all([RuleUsageEvent(**row) for row in rows])
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await _report_failure("write")
|
||||
|
||||
|
||||
def _schedule(rows: list[dict]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
spawn(_insert_events(rows), site="rule_usage_write")
|
||||
|
||||
|
||||
def record_rule_surfaced(
|
||||
*, user_id: int | None, rule_ids: list[int] | set[int], source: str,
|
||||
detail: dict[int, str] | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: record that these rules were shown to the agent.
|
||||
|
||||
Takes the whole delivery at once — one insert per surfacing event, not per
|
||||
rule — because a hint is a single decision and its rows should land
|
||||
together.
|
||||
|
||||
Record what was actually SHOWN, never what was considered. For the ranked
|
||||
arm that means the post-filter hits: it drops what the session already
|
||||
holds (`exclude_rule_ids`) before it speaks, and a rule considered and not
|
||||
shown was not surfaced. Counting those would inflate the denominator with
|
||||
claims the agent never saw, which reads as a precision problem the arm does
|
||||
not have.
|
||||
|
||||
Bulk surfaces pass their whole delivered set, which is the same rule read
|
||||
from the other end — everything in a preload IS shown. `source` is what
|
||||
separates the two afterwards (see `RANKED_SOURCES`); this function does not
|
||||
care which kind it is recording.
|
||||
|
||||
`detail` maps a rule id to what to record beside its row — the moment a
|
||||
mounted rule arrived at, so a mount's pull-through can be
|
||||
read per moment rather than only per arm.
|
||||
"""
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"user_id": user_id,
|
||||
"rule_id": int(rid),
|
||||
"event": SURFACED,
|
||||
"source": source,
|
||||
**({"detail": detail[rid]} if detail and rid in detail else {}),
|
||||
}
|
||||
for rid in rule_ids
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("rule usage payload build failed", exc_info=True)
|
||||
return
|
||||
_schedule(rows)
|
||||
|
||||
|
||||
def record_rule_pulled(*, user_id: int | None, rule_id: int, source: str) -> None:
|
||||
"""Fire-and-forget: record that a rule was opened in full.
|
||||
|
||||
A PULL is somebody choosing to open one record. `enter_project` is NOT a
|
||||
pull — they are bulk resident loads that hand over
|
||||
every applicable rule at once, and counting them would swamp the signal
|
||||
with the very ambient delivery the ratio exists to distinguish from.
|
||||
"""
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"user_id": user_id,
|
||||
"rule_id": int(rule_id),
|
||||
"event": PULLED,
|
||||
"source": source,
|
||||
}
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("rule usage payload build failed", exc_info=True)
|
||||
return
|
||||
_schedule(rows)
|
||||
|
||||
|
||||
def record_rule_outcome(
|
||||
*,
|
||||
user_id: int | None,
|
||||
rule_id: int,
|
||||
outcome: str,
|
||||
source: str,
|
||||
detail: str = "",
|
||||
) -> None:
|
||||
"""Fire-and-forget: record what a rule ACTUALLY CHANGED.
|
||||
|
||||
The third stream. `surfaced` says the system offered a rule; `pulled`
|
||||
says somebody opened it. Neither says whether it made any difference, so
|
||||
without this a rule that fires constantly and is always obeyed and a rule
|
||||
that fires constantly and is never obeyed produce identical telemetry.
|
||||
The second is far the more urgent.
|
||||
|
||||
Two outcomes, because there are only two a judge can honestly report:
|
||||
|
||||
APPLIED — the rule changed what was done, or confirmed it. No `detail`
|
||||
required: following a rule is the unremarkable case and
|
||||
charging prose for it is how an event stops being recorded.
|
||||
DEPARTED — read, and deliberately not followed. `detail` is REQUIRED
|
||||
and is the entire value of the event. A departure without
|
||||
its reason reads back as a miss, which collapses the two
|
||||
states this exists to separate.
|
||||
|
||||
THERE IS NO THIRD CALL, and the absence is the design. Read-and-silently-
|
||||
unchanged is real — it is the failure this stream exists to expose — but
|
||||
it cannot be reported, because an agent that knew it was ignoring a rule
|
||||
would not be ignoring it. It is derived: a rule pulled, with no outcome
|
||||
behind it. See `outcome_state`.
|
||||
|
||||
Guarded rather than trusting: a bad outcome or a reasonless departure is
|
||||
dropped and REPORTED, never written. Telemetry that lies is worse than
|
||||
telemetry that is missing, and a `departed` row with an empty
|
||||
reason is a lie the readout cannot detect.
|
||||
"""
|
||||
if outcome not in OUTCOMES:
|
||||
logger.warning("rule outcome rejected: unknown outcome %r", outcome)
|
||||
spawn(_report_failure("outcome_unknown"), site="rule_usage_outcome")
|
||||
return
|
||||
if outcome == DEPARTED and not (detail or "").strip():
|
||||
logger.warning("rule outcome rejected: departure with no reason")
|
||||
spawn(_report_failure("outcome_no_reason"), site="rule_usage_outcome")
|
||||
return
|
||||
try:
|
||||
rows = [
|
||||
{
|
||||
"user_id": user_id,
|
||||
"rule_id": int(rule_id),
|
||||
"event": outcome,
|
||||
"source": source,
|
||||
"detail": (detail or "").strip() or None,
|
||||
}
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("rule usage payload build failed", exc_info=True)
|
||||
return
|
||||
_schedule(rows)
|
||||
|
||||
|
||||
# What a rule's usage says happened to it, in one word. The four states are
|
||||
# ordered by how much the system actually knows; recorded outcomes are what
|
||||
# keep UNACTED from hiding inside FOLLOWED.
|
||||
UNREAD = "unread" # surfaced, never opened
|
||||
UNACTED = "unacted" # opened, and nothing recorded after — the blind spot
|
||||
FOLLOWED = "followed" # opened and applied
|
||||
DEPARTED_FROM = "departed" # opened and deliberately not followed, with a why
|
||||
|
||||
|
||||
def outcome_state(usage: dict) -> str:
|
||||
"""Which of the four states a rule's usage is in.
|
||||
|
||||
Pure, and reading only the aggregate `usage_for_rules` already returns —
|
||||
so the readout, the badge and any later session summary all answer this
|
||||
question the same way. Two callers computing "was this followed" from raw
|
||||
counts would drift.
|
||||
|
||||
PRECEDENCE, and it is deliberate: a departure outranks an application.
|
||||
A rule both applied and departed from in the same window is a rule
|
||||
someone argued with, and the argument is the interesting half — reporting
|
||||
it as plain compliance would hide the one row a reader most wants.
|
||||
|
||||
UNACTED is the derived state and the reason this function exists. It is
|
||||
not "no data"; it is a rule that was surfaced, deliberately OPENED, and
|
||||
then left no trace of having mattered. That is a much stronger signal
|
||||
than never having been opened at all, and without recorded outcomes it
|
||||
is indistinguishable from compliance.
|
||||
"""
|
||||
if int(usage.get("departed_count") or 0):
|
||||
return DEPARTED_FROM
|
||||
if int(usage.get("applied_count") or 0):
|
||||
return FOLLOWED
|
||||
if int(usage.get("pull_count") or 0):
|
||||
return UNACTED
|
||||
return UNREAD
|
||||
|
||||
|
||||
def empty_rule_usage() -> dict:
|
||||
"""The zero readout — what a rule with no recorded events looks like.
|
||||
|
||||
Callers render this shape unconditionally, so a rule predating the table
|
||||
reads as "never surfaced, never pulled" rather than as a missing key. That
|
||||
distinction matters more here than for notes: every rule in an install
|
||||
predates this table, so for a while "no events" is the normal state and it
|
||||
must not look like a broken readout.
|
||||
|
||||
`surfaced_count` is RANKED surfacings only; `ambient_count` is the bulk
|
||||
deliveries (see `RANKED_SOURCES`). The split is what keeps the badge's
|
||||
"shown often, opened never → dead weight" reading honest: every rule in an
|
||||
always-on set is delivered every session, so an unsplit counter would rank
|
||||
the resident set as the most-surfaced rules in the install purely for being
|
||||
resident.
|
||||
"""
|
||||
return {
|
||||
"surfaced_count": 0,
|
||||
"ambient_count": 0,
|
||||
"pull_count": 0,
|
||||
# The outcome half. Zero here means "nothing recorded", which for a
|
||||
# rule that was also never pulled is simply silence — and for one
|
||||
# that WAS pulled is the blind spot (UNACTED).
|
||||
# `outcome_state` is what tells those apart; no caller should be
|
||||
# reading these counts raw to decide it.
|
||||
#
|
||||
# The REASON for a departure is on the row (`detail`), not here. One
|
||||
# GROUP BY cannot carry the text of the latest departure without a
|
||||
# DISTINCT ON alongside it, and a key that the aggregate could never
|
||||
# fill would read as "no reason given" on every rule that has one —
|
||||
# a permanently-null field that lies. The readout that needs the
|
||||
# prose reads the rows.
|
||||
"applied_count": 0,
|
||||
"departed_count": 0,
|
||||
"last_outcome_at": None,
|
||||
"last_surfaced_at": None,
|
||||
"last_pulled_at": None,
|
||||
}
|
||||
|
||||
|
||||
async def usage_for_rules(rule_ids: list[int]) -> dict[int, dict]:
|
||||
"""Aggregate usage for a set of rules: {rule_id: {counts + timestamps}}.
|
||||
|
||||
One GROUP BY for the whole page rather than a query per row — this feeds a
|
||||
list view, so the per-row shape would be N+1 by construction. Rules with no
|
||||
events come back with `empty_rule_usage()`, so the caller never has to tell
|
||||
"no events" from "not in the result".
|
||||
"""
|
||||
ids = [int(r) for r in rule_ids]
|
||||
out: dict[int, dict] = {rid: empty_rule_usage() for rid in ids}
|
||||
if not ids:
|
||||
return out
|
||||
|
||||
# Classified in SQL so the group stays small: per rule we get at most
|
||||
# (surfaced-ranked, surfaced-ambient, pulled) rather than a row per distinct
|
||||
# source. ONE labelled expression, bound to a variable and reused in the
|
||||
# GROUP BY — a second `case()` instance there renders its own expanding-IN
|
||||
# bind names under asyncpg, so the database sees two DIFFERENT expressions
|
||||
# and rejects the query with a GroupingError. The note twin carries the
|
||||
# same warning for the same reason: the rejection is swallowed and every
|
||||
# counter reads zero while the writes are landing fine.
|
||||
ambient = case(
|
||||
(RuleUsageEvent.source.notin_(RANKED_SOURCES), True),
|
||||
else_=False,
|
||||
).label("ambient")
|
||||
try:
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
RuleUsageEvent.rule_id,
|
||||
RuleUsageEvent.event,
|
||||
func.count().label("n"),
|
||||
func.max(RuleUsageEvent.created_at).label("last_at"),
|
||||
ambient,
|
||||
)
|
||||
.where(RuleUsageEvent.rule_id.in_(ids))
|
||||
.group_by(
|
||||
RuleUsageEvent.rule_id,
|
||||
RuleUsageEvent.event,
|
||||
ambient,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
except Exception:
|
||||
# A telemetry readout must not be able to break the list it decorates —
|
||||
# but it must say it failed, or a broken readout is indistinguishable
|
||||
# from a corpus nobody uses.
|
||||
await _report_failure("readout")
|
||||
return out
|
||||
|
||||
for rule_id, event, n, last_at, is_amb in rows:
|
||||
slot = out.get(int(rule_id))
|
||||
if slot is None:
|
||||
continue
|
||||
# Outcomes first, and never split by ambient. `ambient` asks whether
|
||||
# a RANKER chose to show the rule; an outcome is reported by a judge
|
||||
# after the fact and has no ranker behind it, so the flag is noise
|
||||
# here. Branching on it would silently drop every outcome row into a
|
||||
# bucket nothing reads.
|
||||
if event in OUTCOMES:
|
||||
key = "applied_count" if event == APPLIED else "departed_count"
|
||||
slot[key] = slot[key] + int(n)
|
||||
prev = slot["last_outcome_at"]
|
||||
now = iso(last_at)
|
||||
if now and (prev is None or now > prev):
|
||||
slot["last_outcome_at"] = now
|
||||
elif event == SURFACED and is_amb:
|
||||
slot["ambient_count"] = int(n)
|
||||
elif event == SURFACED:
|
||||
slot["surfaced_count"] = int(n)
|
||||
slot["last_surfaced_at"] = iso(last_at)
|
||||
elif event == PULLED:
|
||||
# Pulls are pulls regardless of what surfaced the rule — "did
|
||||
# anyone ever open this?" does not depend on how it was found. Both
|
||||
# halves accumulate, so this ADDS rather than assigns: a rule can
|
||||
# now be pulled after a ranked hint and after a preload, and the
|
||||
# split arrives as two rows.
|
||||
slot["pull_count"] = slot["pull_count"] + int(n)
|
||||
latest = iso(last_at)
|
||||
if latest and (slot["last_pulled_at"] or "") < latest:
|
||||
slot["last_pulled_at"] = latest
|
||||
return out
|
||||
@@ -0,0 +1,82 @@
|
||||
"""A rule's edit history — the snapshot taken when an edit overwrites text.
|
||||
|
||||
Sibling of `note_versions`, and the differences are the whole design.
|
||||
|
||||
WHAT A VERSION IS FOR. It records what a rule USED TO SAY, because an update
|
||||
overwrites that and nothing can recompute it. It is not a record of the rule's
|
||||
lifecycle: a soft delete keeps the rule row whole and restorable, so there is
|
||||
nothing to preserve at delete time, and anything that outlived a purge would be
|
||||
data the operator explicitly asked to be gone.
|
||||
|
||||
WHAT IS DELIBERATELY NOT COPIED FROM note_versions. Both of its
|
||||
guards defend against note autosave, which fires every 60 seconds and would
|
||||
otherwise burn every slot. Rules have no autosave — every version here comes
|
||||
from a deliberate `update_rule` — so both guards would only ever discard a real
|
||||
edit:
|
||||
|
||||
- No `MIN_VERSION_INTERVAL_SECONDS`. Two deliberate edits four minutes apart
|
||||
are two edits.
|
||||
- No `MAX_VERSIONS` and no pruning. A rule is edited a handful of times in its
|
||||
life; a cap invites losing the one edit somebody needed.
|
||||
|
||||
TEXT ONLY. A rule's topic, its Systems tags and its typed edges are not here.
|
||||
They have their own lifecycle, and folding them in would make one word,
|
||||
"version", mean two things — the rule's wording, and the rule's place in the
|
||||
graph.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.rule_version import RuleVersion
|
||||
|
||||
# The columns a version carries, and therefore the ones whose change is worth
|
||||
# a snapshot. `order_index` and `arose_from_id` are editable through
|
||||
# update_rule and are deliberately absent: reordering a rulebook is not an
|
||||
# edit to what any rule says, and a run of "moved rule 4 above rule 3"
|
||||
# snapshots would bury the edits somebody is actually looking for.
|
||||
SNAPSHOT_FIELDS = (
|
||||
"title", "statement", "why", "how_to_apply", "when_to_apply",
|
||||
"kind", "verify_with", "expires_when",
|
||||
)
|
||||
|
||||
|
||||
def snapshot(rule) -> dict:
|
||||
"""The rule's text right now, as a plain dict.
|
||||
|
||||
Plain values rather than the ORM object because the caller has to hold
|
||||
this ACROSS the mutation — a live `rule` would show the new text by the
|
||||
time it was read, which is the one thing the snapshot must not do.
|
||||
"""
|
||||
return {f: getattr(rule, f) for f in SNAPSHOT_FIELDS}
|
||||
|
||||
|
||||
def record_if_changed(session, rule, actor_user_id: int | None, before: dict):
|
||||
"""Add a version holding `before`, unless the edit changed no text.
|
||||
|
||||
Git semantics, and not a nicety: `update_rule` is reached by callers that
|
||||
resend every field, so without this a form saved twice would file a second
|
||||
identical snapshot and the history would stop reading as a list of edits.
|
||||
|
||||
Session-BOUND, unlike `note_versions.create_version` which opens its own.
|
||||
The version and the edit that caused it commit together or not at all;
|
||||
a separate session could leave a snapshot behind after the update it
|
||||
describes had failed, which is a history entry for an edit that never
|
||||
happened.
|
||||
"""
|
||||
if snapshot(rule) == before:
|
||||
return None
|
||||
version = RuleVersion(
|
||||
rule_id=rule.id, user_id=actor_user_id, **before,
|
||||
)
|
||||
session.add(version)
|
||||
return version
|
||||
|
||||
|
||||
async def list_versions(rule_id: int) -> list[RuleVersion]:
|
||||
"""Newest first — the reader's question is "what did this just say?"."""
|
||||
async with async_session() as session:
|
||||
return list((await session.execute(
|
||||
select(RuleVersion)
|
||||
.where(RuleVersion.rule_id == rule_id)
|
||||
.order_by(RuleVersion.created_at.desc(), RuleVersion.id.desc())
|
||||
)).scalars().all())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
"""One APScheduler job bridged into the asyncio loop — the shape the
|
||||
*_scheduler modules (recurrence spawn, auto-pin scan, trash purge, DB
|
||||
maintenance) share instead of each carrying a private copy.
|
||||
|
||||
APScheduler's BackgroundScheduler fires from a worker thread; the work is
|
||||
async and must run on the app's loop, so the fire is bridged with
|
||||
``run_coroutine_threadsafe``. Each job is a module-level singleton: start is
|
||||
idempotent, stop shuts the scheduler down, and a job whose trigger the
|
||||
operator can change (the maintenance hour) reschedules the live job instead
|
||||
of restarting.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduledJob:
|
||||
"""A named APScheduler job that awaits ``work()`` on the asyncio loop.
|
||||
|
||||
``work`` is an async callable; exceptions it raises are logged under
|
||||
``label`` and never propagate into APScheduler's thread.
|
||||
"""
|
||||
|
||||
def __init__(self, job_id: str, work: Callable[[], Awaitable[None]], *, label: str) -> None:
|
||||
self.job_id = job_id
|
||||
self._work = work
|
||||
self.label = label
|
||||
self._scheduler: BackgroundScheduler | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self._scheduler is not None
|
||||
|
||||
def _fire(self) -> None:
|
||||
"""APScheduler invokes this from its worker thread; bridge into the loop."""
|
||||
if self._loop is None:
|
||||
logger.warning("%s scheduler: no loop registered", self.label)
|
||||
return
|
||||
|
||||
async def _runner() -> None:
|
||||
try:
|
||||
await self._work()
|
||||
except Exception:
|
||||
logger.exception("%s run failed", self.label)
|
||||
|
||||
asyncio.run_coroutine_threadsafe(_runner(), self._loop)
|
||||
|
||||
def start(self, loop: asyncio.AbstractEventLoop, trigger, *, describe: str = "") -> None:
|
||||
"""Start the job on ``trigger``. Idempotent — a second start is a no-op."""
|
||||
if self._scheduler is not None:
|
||||
return
|
||||
self._loop = loop
|
||||
self._scheduler = BackgroundScheduler()
|
||||
self._scheduler.add_job(
|
||||
self._fire, trigger=trigger, id=self.job_id, replace_existing=True,
|
||||
)
|
||||
self._scheduler.start()
|
||||
logger.info("%s scheduler started%s", self.label, f" ({describe})" if describe else "")
|
||||
|
||||
def reschedule(self, trigger, *, describe: str = "") -> None:
|
||||
"""Move the live job to a new trigger; a no-op when not running."""
|
||||
if self._scheduler is None:
|
||||
return
|
||||
self._scheduler.reschedule_job(self.job_id, trigger=trigger)
|
||||
logger.info("%s scheduler rescheduled%s", self.label, f" to {describe}" if describe else "")
|
||||
|
||||
def stop(self) -> None:
|
||||
if self._scheduler is not None:
|
||||
self._scheduler.shutdown(wait=False)
|
||||
self._scheduler = None
|
||||
logger.info("%s scheduler stopped", self.label)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Sealing a credential Grimoire stores so the database alone cannot read it.
|
||||
|
||||
Fernet (AES-128-CBC with an HMAC) under a key derived from the instance's
|
||||
SECRET_KEY. A copy of the database, or of a backup made from it, then holds
|
||||
only ciphertext: reading the credential also needs the SECRET_KEY, which lives
|
||||
in the deployment's environment rather than in the data.
|
||||
|
||||
The derivation is labelled and versioned, so this key is never the same bytes
|
||||
as anything else made from SECRET_KEY (the session-cookie signing uses it
|
||||
raw). An instance that changes its SECRET_KEY can no longer open what it
|
||||
sealed before: `unseal` answers None, and the caller asks for the credential
|
||||
again rather than failing on it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from grimoire.config import Config
|
||||
|
||||
# Every stored secret is sealed under a key derived with this label, so it
|
||||
# keeps the app's earlier name: changing it would leave them unreadable.
|
||||
_LABEL = b"scribe.secrets.v1\x00"
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
digest = hashlib.sha256(_LABEL + Config.SECRET_KEY.encode()).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
|
||||
def seal(plain: str) -> str:
|
||||
"""Encrypt `plain` for storage; the result is ASCII."""
|
||||
return _fernet().encrypt(plain.encode()).decode()
|
||||
|
||||
|
||||
def unseal(token: str) -> str | None:
|
||||
"""Decrypt a value `seal` produced, or None when it can't be opened —
|
||||
sealed under a different SECRET_KEY, or not a sealed value at all."""
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
return _fernet().decrypt(token.encode()).decode()
|
||||
except (InvalidToken, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,121 @@
|
||||
import logging
|
||||
|
||||
from sqlalchemy import delete as sa_delete, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.setting import Setting
|
||||
from grimoire.models.user import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# What a stored credential looks like on the wire. Every surface that READS a
|
||||
# secret (smtp_password, forge_webhook_secret, a forge token) returns this
|
||||
# when one is set; every surface that WRITES one treats this value coming back
|
||||
# as "unchanged", never as a request to store eight asterisks over the real
|
||||
# credential. One constant so the read and write halves cannot disagree.
|
||||
SECRET_MASK = "********"
|
||||
|
||||
|
||||
def bounded_float(raw: str | None, default: float, lo: float = 0.0, hi: float = 1.0) -> float:
|
||||
"""A numeric setting as stored text, parsed and clamped to [lo, hi].
|
||||
|
||||
Every similarity bar and retrieval floor is kept as a string and read the
|
||||
same way: an unparseable value falls back to the DEFAULT, never to 0 (a
|
||||
floor of 0 admits everything), and a value out of range is pulled back
|
||||
into it. Pure on purpose: each caller keeps its own `get_setting` read, so
|
||||
what can fail there — and whether that fails open — stays the caller's.
|
||||
"""
|
||||
try:
|
||||
value = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return min(hi, max(lo, value))
|
||||
|
||||
|
||||
async def get_admin_setting(key: str, default: str = "") -> str:
|
||||
"""Read an instance-global setting (one stored on an admin account).
|
||||
|
||||
Used for settings that aren't per-user — e.g. the plugin marketplace URL
|
||||
shown to everyone in Settings. Mirrors the admin-scoped lookup used for
|
||||
the application base URL.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting)
|
||||
.join(User, Setting.user_id == User.id)
|
||||
.where(User.role == "admin", Setting.key == key)
|
||||
)
|
||||
setting = result.scalars().first()
|
||||
return setting.value if setting and setting.value else default
|
||||
|
||||
|
||||
async def set_admin_setting(key: str, value: str) -> None:
|
||||
"""Write an instance-global setting onto the first admin account.
|
||||
|
||||
The write-side counterpart to get_admin_setting, for non-per-user settings
|
||||
written outside a request context (e.g. a scheduler persisting its last-run
|
||||
summary) where get_current_user_id() isn't available.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
admin_id = (
|
||||
await session.execute(
|
||||
select(User.id).where(User.role == "admin").order_by(User.id)
|
||||
)
|
||||
).scalars().first()
|
||||
if admin_id is not None:
|
||||
await set_setting(admin_id, key, value)
|
||||
|
||||
|
||||
async def get_setting(user_id: int, key: str, default: str = "") -> str:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting).where(Setting.user_id == user_id, Setting.key == key)
|
||||
)
|
||||
setting = result.scalar_one_or_none()
|
||||
return setting.value if setting else default
|
||||
|
||||
|
||||
async def set_setting(user_id: int, key: str, value: str) -> None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting).where(Setting.user_id == user_id, Setting.key == key)
|
||||
)
|
||||
setting = result.scalar_one_or_none()
|
||||
if setting:
|
||||
setting.value = value
|
||||
else:
|
||||
session.add(Setting(user_id=user_id, key=key, value=value))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def set_settings_batch(user_id: int, settings: dict[str, str]) -> None:
|
||||
"""Update multiple settings in a single transaction."""
|
||||
async with async_session() as session:
|
||||
for key, value in settings.items():
|
||||
result = await session.execute(
|
||||
select(Setting).where(Setting.user_id == user_id, Setting.key == key)
|
||||
)
|
||||
setting = result.scalar_one_or_none()
|
||||
if setting:
|
||||
setting.value = value
|
||||
else:
|
||||
session.add(Setting(user_id=user_id, key=key, value=value))
|
||||
await session.commit()
|
||||
logger.info("Batch-updated %d settings for user %d", len(settings), user_id)
|
||||
|
||||
|
||||
async def delete_setting(user_id: int, key: str) -> None:
|
||||
"""Remove a setting row so get_setting() returns its hardcoded default instead."""
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
sa_delete(Setting).where(Setting.user_id == user_id, Setting.key == key)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_all_settings(user_id: int) -> dict[str, str]:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(Setting).where(Setting.user_id == user_id)
|
||||
)
|
||||
return {s.key: s.value for s in result.scalars().all()}
|
||||
@@ -0,0 +1,191 @@
|
||||
"""The end-of-turn shape check: what a turn wrote that nobody has judged, and what the agent is asked.
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
The agent that WROTE the code is the one participant who knows what it is.
|
||||
Machinery can find definitions and stamp "instance of #N", but it cannot say
|
||||
that a first instance is a reusable piece — and a piece never recorded is
|
||||
rebuilt by the next session. An audit run later asks too late, if anyone runs
|
||||
one.
|
||||
|
||||
So the question is asked at the moment the knowledge exists. The client's write
|
||||
hooks keep a ledger of the definitions each turn wrote; its Stop hook sends
|
||||
them here at the end of the turn. Two jobs live on this side, as with the
|
||||
report check (services/report_check.py):
|
||||
|
||||
- DECIDING what is unjudged, against the ledger and the project's recorded
|
||||
snippets — something only the server can see.
|
||||
- OWNING THE WORDS. A hook carries timing and transport (plugin/PACKAGING.md);
|
||||
the instruction comes from here, so every client asks the same question
|
||||
and the wording changes in one place.
|
||||
|
||||
And it RECORDS every outcome, so "how much did agents leave unjudged after
|
||||
being asked" is a number rather than an impression. app_logs, for the reason
|
||||
report_check gives.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.app_log import AppLog
|
||||
|
||||
# check — the turn's first stop: block if anything is unjudged.
|
||||
# after — the stop that follows a block: never block again, only record
|
||||
# whether the agent answered.
|
||||
PHASES = ("check", "after")
|
||||
OUTCOMES = ("passed", "blocked", "judged_after_block", "left_after_block")
|
||||
|
||||
# How many shapes the reason names before "+N more". Enough to judge a normal
|
||||
# turn in one call; a turn that wrote more than this generated something.
|
||||
_LISTED = 25
|
||||
|
||||
|
||||
def _evidence_note(evidence: dict) -> str:
|
||||
"""One parenthesis of what the machinery holds — offered, never asserted."""
|
||||
parts = []
|
||||
looks = evidence.get("looks_like") or {}
|
||||
if looks.get("snippet_id"):
|
||||
score = looks.get("score")
|
||||
at = f" {score:.2f}" if isinstance(score, (int, float)) else ""
|
||||
parts.append(f"looks like #{looks['snippet_id']}{at}")
|
||||
stamped = evidence.get("stamped") or {}
|
||||
if stamped.get("snippet_id"):
|
||||
parts.append(f"the hook guessed #{stamped['snippet_id']}")
|
||||
if evidence.get("copies"):
|
||||
parts.append("other copies of it exist")
|
||||
if evidence.get("diverges_from"):
|
||||
parts.append(f"#{evidence['diverges_from']} is canon in this directory")
|
||||
return f" ({'; '.join(parts)})" if parts else ""
|
||||
|
||||
|
||||
def block_reason(unjudged: list[dict], *, project_id: int, repo: str) -> str:
|
||||
"""What the agent is told at the end of a turn that wrote unjudged shapes.
|
||||
|
||||
Phrased as a practice: what to decide and the one
|
||||
call that records it, not a prohibition. The reusing-code skill owns the
|
||||
longer version of what each verdict means.
|
||||
"""
|
||||
lines = []
|
||||
for item in unjudged[:_LISTED]:
|
||||
new = " — new" if item.get("status") == "new" else ""
|
||||
lines.append(
|
||||
f"- {item['path']} · {item['symbol']} ({item['kind']}){new}"
|
||||
f"{_evidence_note(item.get('evidence') or {})}"
|
||||
)
|
||||
more = len(unjudged) - _LISTED
|
||||
if more > 0:
|
||||
lines.append(f"- … and {more} more (list_shapes(project_id={project_id}, path=…) shows them)")
|
||||
repo_arg = f', repo="{repo}"' if repo else ""
|
||||
return (
|
||||
f"This turn wrote {len(unjudged)} definition{'s' if len(unjudged) != 1 else ''} "
|
||||
"that nobody has judged yet. You built them, so you know what each one is — "
|
||||
"record it now, while that is still true, so the next session starts from your "
|
||||
"answer instead of rebuilding it:\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nFor each: something another part of the code should reuse → record it "
|
||||
"with create_snippet (name, code, when to reach for it, location); built from a "
|
||||
"recorded snippet → `instance` of it; a deliberate departure from one → "
|
||||
"`variant` with the why; a genuine one-off → `exempt` with a reason "
|
||||
"(reason_code one-off-handler, test-helper, scoped-css, pure-helper … indexes it). "
|
||||
"One call records them all: "
|
||||
f"classify_shapes(project_id={project_id}{repo_arg}, classifications=[{{path, "
|
||||
"symbol, kind, status, snippet_id?, reason?, reason_code?}}, …]). "
|
||||
"`repo` lets a verdict land on a shape the ledger has not synced yet."
|
||||
)
|
||||
|
||||
|
||||
async def record_shape_check(
|
||||
user_id: int | None,
|
||||
outcome: str,
|
||||
*,
|
||||
written: int,
|
||||
unjudged: int,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
if outcome not in OUTCOMES:
|
||||
raise ValueError(f"unknown shape-check outcome {outcome!r}")
|
||||
details: dict = {"outcome": outcome, "written": written, "unjudged": unjudged}
|
||||
if project_id:
|
||||
details["project_id"] = project_id
|
||||
async with async_session() as session:
|
||||
session.add(AppLog(
|
||||
category="plugin",
|
||||
user_id=user_id,
|
||||
action="shape_check",
|
||||
details=json.dumps(details),
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
|
||||
def outcome_for(phase: str, unjudged: list[dict]) -> str:
|
||||
"""The recorded outcome: a first stop blocks or passes; the stop after a
|
||||
block records whether the agent answered, and never blocks."""
|
||||
if phase == "after":
|
||||
return "left_after_block" if unjudged else "judged_after_block"
|
||||
return "blocked" if unjudged else "passed"
|
||||
|
||||
|
||||
def parse_written(raw: str, *, cap: int = 200) -> list[tuple[str, str, str]]:
|
||||
"""The hook's ledger lines, `path<TAB>kind<TAB>name`, as triples.
|
||||
|
||||
Kinds outside the ledger's vocabulary and malformed lines are dropped
|
||||
rather than echoed into an instruction; duplicates collapse; capped."""
|
||||
out: list[tuple[str, str, str]] = []
|
||||
for line in (raw or "").splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
path, kind, name = (p.strip() for p in parts)
|
||||
if kind not in ("css", "sym", "file") or not path or not name:
|
||||
continue
|
||||
if (path, kind, name) not in out:
|
||||
out.append((path, kind, name))
|
||||
if len(out) >= cap:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
# The window the coverage line reports the write-time figure over. A week is
|
||||
# long enough to hold a few working sessions on a project and short enough
|
||||
# that a reflex that started slipping shows up while it is still news.
|
||||
WINDOW_DAYS = 7
|
||||
|
||||
|
||||
def summarise(outcomes: list[str]) -> dict:
|
||||
"""{checked, asked, left} from a list of recorded outcomes.
|
||||
|
||||
`checked` is turns the question was put to (a first stop that wrote
|
||||
something): passed + blocked. `asked` is the blocked ones. `left` is
|
||||
asks the agent walked past — the stop after a block that still had
|
||||
unjudged shapes. That last number is the slip this check exists to make
|
||||
visible."""
|
||||
checked = sum(1 for o in outcomes if o in ("passed", "blocked"))
|
||||
asked = sum(1 for o in outcomes if o == "blocked")
|
||||
left = sum(1 for o in outcomes if o == "left_after_block")
|
||||
return {"checked": checked, "asked": asked, "left": left}
|
||||
|
||||
|
||||
async def window_summary(project_id: int, *, days: int = WINDOW_DAYS) -> dict:
|
||||
"""The write-time figure for one project over the last ``days``."""
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(AppLog.details).where(
|
||||
AppLog.category == "plugin",
|
||||
AppLog.action == "shape_check",
|
||||
AppLog.created_at >= since,
|
||||
)
|
||||
)).scalars().all()
|
||||
outcomes: list[str] = []
|
||||
for raw in rows:
|
||||
try:
|
||||
details = json.loads(raw or "{}")
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if details.get("project_id") == project_id:
|
||||
outcomes.append(details.get("outcome") or "")
|
||||
return {**summarise(outcomes), "days": days}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,262 @@
|
||||
"""Sharing service — create/manage project and note shares."""
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.group import Group, GroupMembership
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.share import NoteShare, ProjectShare
|
||||
from grimoire.models.user import User
|
||||
from grimoire.models.base import iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_PERMISSIONS = {"viewer", "editor", "admin"}
|
||||
|
||||
|
||||
async def _enrich_shares(session, shares) -> list[dict]:
|
||||
"""Attach username / group_name to a list of share rows (ProjectShare or NoteShare)."""
|
||||
result = []
|
||||
for s in shares:
|
||||
d = s.to_dict()
|
||||
if s.shared_with_user_id:
|
||||
user = await session.get(User, s.shared_with_user_id)
|
||||
d["username"] = user.username if user else None
|
||||
if s.shared_with_group_id:
|
||||
group = await session.get(Group, s.shared_with_group_id)
|
||||
d["group_name"] = group.name if group else None
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
|
||||
def best_permission_by(shares, id_attr: str) -> dict[int, str]:
|
||||
"""Return {resource_id: best_permission} keeping the highest-ranked permission per resource."""
|
||||
from grimoire.services.access import PERMISSION_RANK
|
||||
seen: dict[int, str] = {}
|
||||
for share in shares:
|
||||
rid = getattr(share, id_attr)
|
||||
prev = seen.get(rid)
|
||||
if prev is None or PERMISSION_RANK[share.permission] > PERMISSION_RANK[prev]:
|
||||
seen[rid] = share.permission
|
||||
return seen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project shares
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def share_project(
|
||||
acting_user_id: int,
|
||||
project_id: int,
|
||||
*,
|
||||
target_user_id: int | None = None,
|
||||
target_group_id: int | None = None,
|
||||
permission: str = "viewer",
|
||||
) -> ProjectShare | None:
|
||||
if permission not in VALID_PERMISSIONS:
|
||||
return None
|
||||
from grimoire.services.access import can_admin_project
|
||||
if not await can_admin_project(acting_user_id, project_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
share = ProjectShare(
|
||||
project_id=project_id,
|
||||
shared_with_user_id=target_user_id,
|
||||
shared_with_group_id=target_group_id,
|
||||
permission=permission,
|
||||
invited_by=acting_user_id,
|
||||
)
|
||||
session.add(share)
|
||||
await session.commit()
|
||||
await session.refresh(share)
|
||||
return share
|
||||
|
||||
|
||||
async def update_project_share(
|
||||
acting_user_id: int, share_id: int, permission: str
|
||||
) -> ProjectShare | None:
|
||||
if permission not in VALID_PERMISSIONS:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
share = await session.get(ProjectShare, share_id)
|
||||
if not share:
|
||||
return None
|
||||
from grimoire.services.access import can_admin_project
|
||||
if not await can_admin_project(acting_user_id, share.project_id):
|
||||
return None
|
||||
share.permission = permission
|
||||
await session.commit()
|
||||
await session.refresh(share)
|
||||
return share
|
||||
|
||||
|
||||
async def remove_project_share(acting_user_id: int, share_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
share = await session.get(ProjectShare, share_id)
|
||||
if not share:
|
||||
return False
|
||||
from grimoire.services.access import can_admin_project
|
||||
if not await can_admin_project(acting_user_id, share.project_id):
|
||||
return False
|
||||
await session.delete(share)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_project_shares(project_id: int) -> list[dict]:
|
||||
async with async_session() as session:
|
||||
shares = (await session.execute(
|
||||
select(ProjectShare).where(ProjectShare.project_id == project_id)
|
||||
)).scalars().all()
|
||||
return await _enrich_shares(session, shares)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Note shares
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def share_note(
|
||||
acting_user_id: int,
|
||||
note_id: int,
|
||||
*,
|
||||
target_user_id: int | None = None,
|
||||
target_group_id: int | None = None,
|
||||
permission: str = "viewer",
|
||||
) -> NoteShare | None:
|
||||
if permission not in VALID_PERMISSIONS:
|
||||
return None
|
||||
from grimoire.services.access import get_note_permission
|
||||
perm = await get_note_permission(acting_user_id, note_id)
|
||||
if perm not in ("admin", "owner"):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
share = NoteShare(
|
||||
note_id=note_id,
|
||||
shared_with_user_id=target_user_id,
|
||||
shared_with_group_id=target_group_id,
|
||||
permission=permission,
|
||||
invited_by=acting_user_id,
|
||||
)
|
||||
session.add(share)
|
||||
await session.commit()
|
||||
await session.refresh(share)
|
||||
return share
|
||||
|
||||
|
||||
async def update_note_share(
|
||||
acting_user_id: int, share_id: int, permission: str
|
||||
) -> NoteShare | None:
|
||||
if permission not in VALID_PERMISSIONS:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
share = await session.get(NoteShare, share_id)
|
||||
if not share:
|
||||
return None
|
||||
from grimoire.services.access import get_note_permission
|
||||
perm = await get_note_permission(acting_user_id, share.note_id)
|
||||
if perm not in ("admin", "owner"):
|
||||
return None
|
||||
share.permission = permission
|
||||
await session.commit()
|
||||
await session.refresh(share)
|
||||
return share
|
||||
|
||||
|
||||
async def remove_note_share(acting_user_id: int, share_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
share = await session.get(NoteShare, share_id)
|
||||
if not share:
|
||||
return False
|
||||
from grimoire.services.access import get_note_permission
|
||||
perm = await get_note_permission(acting_user_id, share.note_id)
|
||||
if perm not in ("admin", "owner"):
|
||||
return False
|
||||
await session.delete(share)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_note_shares(note_id: int) -> list[dict]:
|
||||
async with async_session() as session:
|
||||
shares = (await session.execute(
|
||||
select(NoteShare).where(NoteShare.note_id == note_id)
|
||||
)).scalars().all()
|
||||
return await _enrich_shares(session, shares)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared-with-me
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def list_shared_with_me(user_id: int) -> dict:
|
||||
"""All projects and notes shared with the user (directly or via group)."""
|
||||
async with async_session() as session:
|
||||
user_group_ids = (await session.execute(
|
||||
select(GroupMembership.group_id).where(GroupMembership.user_id == user_id)
|
||||
)).scalars().all()
|
||||
|
||||
# --- Projects ---
|
||||
proj_direct = (await session.execute(
|
||||
select(ProjectShare).where(ProjectShare.shared_with_user_id == user_id)
|
||||
)).scalars().all()
|
||||
|
||||
proj_group: list[ProjectShare] = []
|
||||
if user_group_ids:
|
||||
proj_group = (await session.execute(
|
||||
select(ProjectShare).where(
|
||||
ProjectShare.shared_with_group_id.in_(user_group_ids)
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
seen_projects = best_permission_by(list(proj_direct) + list(proj_group), "project_id")
|
||||
|
||||
projects = []
|
||||
for pid, perm in seen_projects.items():
|
||||
proj = await session.get(Project, pid)
|
||||
if proj:
|
||||
owner = await session.get(User, proj.user_id)
|
||||
projects.append({
|
||||
"id": proj.id,
|
||||
"title": proj.title,
|
||||
"description": proj.description,
|
||||
"status": proj.status,
|
||||
"color": proj.color,
|
||||
"updated_at": iso(proj.updated_at),
|
||||
"owner_username": owner.username if owner else None,
|
||||
"permission": perm,
|
||||
})
|
||||
|
||||
# --- Notes / Tasks ---
|
||||
note_direct = (await session.execute(
|
||||
select(NoteShare).where(NoteShare.shared_with_user_id == user_id)
|
||||
)).scalars().all()
|
||||
|
||||
note_group: list[NoteShare] = []
|
||||
if user_group_ids:
|
||||
note_group = (await session.execute(
|
||||
select(NoteShare).where(
|
||||
NoteShare.shared_with_group_id.in_(user_group_ids)
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
seen_notes = best_permission_by(list(note_direct) + list(note_group), "note_id")
|
||||
|
||||
notes = []
|
||||
for nid, perm in seen_notes.items():
|
||||
note = await session.get(Note, nid)
|
||||
if note:
|
||||
owner = await session.get(User, note.user_id)
|
||||
notes.append({
|
||||
"id": note.id,
|
||||
"title": note.title,
|
||||
"is_task": note.is_task,
|
||||
"project_id": note.project_id,
|
||||
"updated_at": iso(note.updated_at),
|
||||
"owner_username": owner.username if owner else None,
|
||||
"permission": perm,
|
||||
})
|
||||
|
||||
return {"projects": projects, "notes": notes}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,215 @@
|
||||
"""Which records have been overtaken by which — the claim, not the ranking.
|
||||
|
||||
This module only records and reads the relation; the
|
||||
demotion that makes it matter lives in the retrieval layer.
|
||||
|
||||
WHY THE CLAIM POINTS FORWARD
|
||||
|
||||
The note being written declares what it supersedes. The older record cannot
|
||||
know it has been overtaken — asking it to record its own obsolescence is asking
|
||||
it to predict the future. So the party with the knowledge makes the claim, and
|
||||
"has this been superseded?" is derived by looking at the far end.
|
||||
|
||||
WHAT IT MEANS
|
||||
|
||||
A claim, never a proof. It demotes a record in ranked retrieval; it does not
|
||||
assert the older record was wrong and it never hides it. A note that accurately
|
||||
described how something worked in June is still accurate about June.
|
||||
|
||||
Partial and many-to-many by nature: one note may supersede parts of several
|
||||
others, and be overtaken piecemeal by several later ones.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.note_supersession import NoteSupersession
|
||||
from grimoire.services import access
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _closes_a_cycle(session, superseder_id: int, superseded_id: int) -> bool:
|
||||
"""True if `superseder -> superseded` would complete a loop.
|
||||
|
||||
Walks the existing graph from `superseded_id` following superseder→superseded
|
||||
edges. If the walk reaches `superseder_id`, the new edge closes a cycle.
|
||||
|
||||
Why refuse rather than tolerate: a cycle claims every member is obsolete, and
|
||||
under FLAT demotion (see the milestone) that demotes all of them equally —
|
||||
so a set of records that supersede each other in a ring would vanish from
|
||||
ranked retrieval together, which is the opposite of the intent. Nothing about
|
||||
the data would say why.
|
||||
|
||||
Iterative with a visited set, not recursion: the graph is user-supplied and
|
||||
a deep chain must not become a stack overflow in a write path.
|
||||
"""
|
||||
seen: set[int] = set()
|
||||
frontier = [superseded_id]
|
||||
while frontier:
|
||||
current = frontier.pop()
|
||||
if current == superseder_id:
|
||||
return True
|
||||
if current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
rows = (await session.execute(
|
||||
select(NoteSupersession.superseded_id)
|
||||
.where(NoteSupersession.superseder_id == current)
|
||||
)).scalars().all()
|
||||
frontier.extend(int(r) for r in rows)
|
||||
return False
|
||||
|
||||
|
||||
async def set_supersedes(
|
||||
user_id: int, note_id: int, superseded_ids: list[int]
|
||||
) -> list[int] | None:
|
||||
"""Replace what `note_id` claims to supersede (set semantics).
|
||||
|
||||
Returns the resulting list, or None if the caller cannot write the note
|
||||
making the claim.
|
||||
|
||||
WHAT IS SILENTLY DROPPED, and why each is a drop rather than an error:
|
||||
- ids that don't exist or are trashed — the claim has no subject
|
||||
- the note's own id — meaningless, and the DB CHECK would refuse it anyway
|
||||
- an id that would close a cycle — see _closes_a_cycle
|
||||
|
||||
WHAT IS REFUSED OUTRIGHT: a target the caller cannot WRITE. That is not a
|
||||
silent drop, because it is the one case where the caller might reasonably
|
||||
believe they succeeded and be wrong in a way that matters — demoting someone
|
||||
else's record out of their retrieval is damage you cannot see from the
|
||||
outside.
|
||||
"""
|
||||
if not await access.can_write_note(user_id, note_id):
|
||||
return None
|
||||
|
||||
async with async_session() as session:
|
||||
wanted: list[int] = []
|
||||
for target in dict.fromkeys(superseded_ids): # de-dup, keep order
|
||||
target = int(target)
|
||||
if target == note_id:
|
||||
continue
|
||||
note = await session.get(Note, target)
|
||||
if note is None or note.deleted_at is not None:
|
||||
continue
|
||||
if not await access.can_write_note(user_id, target):
|
||||
raise PermissionError(
|
||||
f"note {target} is not yours to supersede — you need write "
|
||||
f"access to it, not just read. Superseding demotes a record "
|
||||
f"in its owner's retrieval too."
|
||||
)
|
||||
if await _closes_a_cycle(session, note_id, target):
|
||||
continue
|
||||
wanted.append(target)
|
||||
|
||||
existing = set((await session.execute(
|
||||
select(NoteSupersession.superseded_id)
|
||||
.where(NoteSupersession.superseder_id == note_id)
|
||||
)).scalars().all())
|
||||
wanted_set = set(wanted)
|
||||
|
||||
to_remove = existing - wanted_set
|
||||
if to_remove:
|
||||
await session.execute(
|
||||
delete(NoteSupersession).where(
|
||||
NoteSupersession.superseder_id == note_id,
|
||||
NoteSupersession.superseded_id.in_(to_remove),
|
||||
)
|
||||
)
|
||||
for target in wanted:
|
||||
if target not in existing:
|
||||
session.add(
|
||||
NoteSupersession(superseder_id=note_id, superseded_id=target)
|
||||
)
|
||||
await session.commit()
|
||||
return wanted
|
||||
|
||||
|
||||
async def get_relations(user_id: int, note_id: int) -> dict[str, list[int]]:
|
||||
"""Both directions for one note: what it supersedes, and what supersedes it.
|
||||
|
||||
ONE query and ONE ACL check, because this runs on every note read. Asking
|
||||
the two questions separately doubled the round trips on the hottest path in
|
||||
the product to save a two-line partition — the wrong trade, and one I made
|
||||
on the first attempt.
|
||||
|
||||
Returns {"supersedes": [...], "superseded_by": [...]}, both sorted. Empty
|
||||
lists when the caller cannot read the note.
|
||||
|
||||
`superseded_by` is the direction that matters to a READER and the one the
|
||||
note itself cannot know. An agent handed a stale record with no marker acts
|
||||
on it confidently, which is worse than never surfacing it at all.
|
||||
"""
|
||||
empty: dict[str, list[int]] = {"supersedes": [], "superseded_by": []}
|
||||
if not await access.can_read_note(user_id, note_id):
|
||||
return empty
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(
|
||||
NoteSupersession.superseder_id, NoteSupersession.superseded_id
|
||||
).where(
|
||||
(NoteSupersession.superseder_id == note_id)
|
||||
| (NoteSupersession.superseded_id == note_id)
|
||||
)
|
||||
)).all()
|
||||
supersedes = sorted(
|
||||
int(old) for new, old in rows if int(new) == note_id
|
||||
)
|
||||
superseded_by = sorted(
|
||||
int(new) for new, old in rows if int(old) == note_id
|
||||
)
|
||||
return {"supersedes": supersedes, "superseded_by": superseded_by}
|
||||
|
||||
|
||||
async def superseded_ids(note_ids: list[int]) -> set[int]:
|
||||
"""Of `note_ids`, which have been superseded by anything. One query.
|
||||
|
||||
Deliberately NOT ACL-scoped: this feeds ranking over a candidate set the
|
||||
caller has already been authorised to see, and re-checking per candidate
|
||||
would be a per-result round trip on a hot path. Callers must pass an
|
||||
already-scoped set — which is why this takes ids rather than a user.
|
||||
"""
|
||||
if not note_ids:
|
||||
return set()
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(NoteSupersession.superseded_id)
|
||||
.where(NoteSupersession.superseded_id.in_(note_ids))
|
||||
)).scalars().all()
|
||||
return {int(r) for r in rows}
|
||||
|
||||
|
||||
SUPERSEDED_HINT = (
|
||||
"A later note claims to bring this up to date — see superseded_by. "
|
||||
"Read this as what was true when written, and check the newer one "
|
||||
"before acting on it."
|
||||
)
|
||||
|
||||
|
||||
async def attach_relations(user_id: int, note_id: int, data: dict, *, hint: bool = False) -> None:
|
||||
"""Add both directions of the supersession relation to a note payload.
|
||||
|
||||
ONE seam for the REST and MCP surfaces, which must agree about what a
|
||||
note's payload says — or the web UI and the agent would disagree about
|
||||
whether a record is current. Both directions, because they answer
|
||||
different questions and only one is obvious: `supersedes` is what the
|
||||
author claimed; `superseded_by` is what a READER needs and what the note
|
||||
itself cannot know — a stale record handed over without that marker gets
|
||||
acted on confidently, which is worse than never surfacing it.
|
||||
|
||||
Omitted entirely when empty, so an ordinary note's payload doesn't grow
|
||||
two permanently-empty lists — a field that always says nothing trains
|
||||
readers to skip fields. `hint=True` (the agent surface) also
|
||||
attaches `superseded_note`, the one-sentence reading instruction.
|
||||
"""
|
||||
rel = await get_relations(user_id, note_id)
|
||||
if rel["supersedes"]:
|
||||
data["supersedes"] = rel["supersedes"]
|
||||
if rel["superseded_by"]:
|
||||
data["superseded_by"] = rel["superseded_by"]
|
||||
if hint:
|
||||
data["superseded_note"] = SUPERSEDED_HINT
|
||||
@@ -0,0 +1,196 @@
|
||||
"""An area's rulings, delivered when its files are touched.
|
||||
|
||||
A RULING is the operator's decision about how one area of the work must behave.
|
||||
It lives in a `Rulings` section at the end of the area's System description
|
||||
(writing-records.md says how one is written). The description already rides
|
||||
every record filed under the System; this module is the other half: the first
|
||||
command or edit in a session that touches the System's files
|
||||
(`System.path_patterns`) shows its rulings, one line per System.
|
||||
|
||||
A LOOKUP, NOT A SEARCH. A shell command scores against prose decisions at noise
|
||||
level, so nothing here is ranked: a path either falls under a System's
|
||||
patterns or it does not. It takes no budget and no floor from the retrieval
|
||||
arms beside it, and writes no retrieval_logs row — there is no score
|
||||
distribution for it to join. Its surfacings go to `system_usage_events`.
|
||||
|
||||
ONCE IN FULL, THEN A REFERENCE. The hook keeps the Systems
|
||||
already shown this session and passes them back; a repeat renders as one short
|
||||
line rather than the rulings again, and is not counted as a surfacing.
|
||||
|
||||
Fails open everywhere: a delivery aid must never break the act it rides on.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import posixpath
|
||||
import re
|
||||
import shlex
|
||||
|
||||
from grimoire.services import systems as systems_svc
|
||||
from grimoire.services.system_usage import record_system_surfaced
|
||||
from grimoire.services.text import elide
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The section heading: `Rulings`, optionally as a markdown heading, bold, or
|
||||
# with a trailing colon. The LAST one wins — the section sits at the end.
|
||||
_HEADING = re.compile(r"^[ \t]*(?:#{1,6}[ \t]*)?\**Rulings\**[ \t]*:?[ \t]*$", re.I | re.M)
|
||||
_BULLET = re.compile(r"^[ \t]*[-*][ \t]+(.*)$")
|
||||
|
||||
# Per line, so one System with a long list cannot crowd the others out.
|
||||
RULING_CHARS = 300
|
||||
RULINGS_PER_SYSTEM = 8
|
||||
|
||||
# A command is not a file list: these bound how much of it is read as paths.
|
||||
_MAX_PATHS = 40
|
||||
_LINE_SUFFIX = re.compile(r":\d+(?::\d+)?$")
|
||||
_HAS_EXTENSION = re.compile(r"\.[A-Za-z0-9]{1,10}$")
|
||||
_ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
|
||||
|
||||
|
||||
def parse_rulings(description: str | None) -> list[str]:
|
||||
"""The rulings in a System description, one string per ruling.
|
||||
|
||||
A bullet under the last `Rulings` heading is a ruling; an indented line
|
||||
after one continues it. The section ends at the first line that is
|
||||
neither — a heading or paragraph written after it is not a ruling.
|
||||
"""
|
||||
text = description or ""
|
||||
found = list(_HEADING.finditer(text))
|
||||
if not found:
|
||||
return []
|
||||
out: list[str] = []
|
||||
for line in text[found[-1].end():].splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
bullet = _BULLET.match(line)
|
||||
if bullet:
|
||||
if bullet.group(1).strip():
|
||||
out.append(bullet.group(1).strip())
|
||||
elif out and line[:1] in (" ", "\t"):
|
||||
out[-1] = f"{out[-1]} {line.strip()}"
|
||||
else:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _relative(token: str, root: str, cwd_rel: str) -> str:
|
||||
"""`token` as a repo-relative path, or "" when it is not one."""
|
||||
if token.startswith("/"):
|
||||
if not root or not token.startswith(root.rstrip("/") + "/"):
|
||||
return ""
|
||||
token = token[len(root.rstrip("/")) + 1:]
|
||||
elif cwd_rel:
|
||||
token = f"{cwd_rel}/{token}"
|
||||
path = posixpath.normpath(token)
|
||||
if path in (".", "") or path == ".." or path.startswith("../"):
|
||||
return ""
|
||||
return path
|
||||
|
||||
|
||||
def command_paths(command: str, *, root: str = "", cwd: str = "") -> list[str]:
|
||||
"""The repo-relative paths a shell command names, in order, deduplicated.
|
||||
|
||||
Reads and writes alike: a session forms its picture of an area by reading
|
||||
it, which is exactly when that area's rulings should reach it. A token is
|
||||
taken as a path when it has a `/` or a file extension; flags, URLs and
|
||||
globs are not paths. `root` is the repo's absolute root and `cwd` the
|
||||
command's working directory, both from the hook — an absolute path
|
||||
outside the root, or a relative one that climbs out of it, is dropped.
|
||||
"""
|
||||
command = command or ""
|
||||
try:
|
||||
tokens = shlex.split(command, comments=False, posix=True)
|
||||
except ValueError:
|
||||
tokens = command.split()
|
||||
root = (root or "").rstrip("/")
|
||||
cwd_rel = ""
|
||||
if root and cwd:
|
||||
cwd = cwd.rstrip("/")
|
||||
if cwd.startswith(root + "/"):
|
||||
cwd_rel = cwd[len(root) + 1:]
|
||||
out: list[str] = []
|
||||
for raw in tokens:
|
||||
token = raw.strip().strip("'\"").rstrip(";,)|&")
|
||||
if token.startswith("-"):
|
||||
if "=" not in token:
|
||||
continue
|
||||
token = token.split("=", 1)[1]
|
||||
elif _ASSIGNMENT.match(token):
|
||||
token = token.split("=", 1)[1]
|
||||
if not token or "://" in token or any(c in token for c in "*?[]{}$`<>"):
|
||||
continue
|
||||
# `file.py:120` and a test id `file.py::test_x` name the file.
|
||||
token = _LINE_SUFFIX.sub("", token.split("::", 1)[0])
|
||||
if "/" not in token and not _HAS_EXTENSION.search(token):
|
||||
continue
|
||||
path = _relative(token, root, cwd_rel)
|
||||
if path and path not in out:
|
||||
out.append(path)
|
||||
if len(out) >= _MAX_PATHS:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _full_line(system, path: str, rulings: list[str]) -> str:
|
||||
shown = [elide(r, RULING_CHARS)[0] for r in rulings[:RULINGS_PER_SYSTEM]]
|
||||
more = len(rulings) - len(shown)
|
||||
listed = " ".join(f"({i}) {r}" for i, r in enumerate(shown, 1))
|
||||
if more > 0:
|
||||
listed += f" (+{more} more in `get_system({system.id})`)"
|
||||
return (
|
||||
f"> Rulings for `{path}` — the operator's decisions about "
|
||||
f"{system.name} (System {system.id}): {listed} "
|
||||
"Work here keeps to them; something that would depart from one is a "
|
||||
"question for the operator, not an option to choose. "
|
||||
"(Shown once per session.)"
|
||||
)
|
||||
|
||||
|
||||
def _reference_line(system, path: str) -> str:
|
||||
return (
|
||||
f"> `{path}` is in {system.name} (System {system.id}) — its rulings "
|
||||
"were shown earlier this session and still apply."
|
||||
)
|
||||
|
||||
|
||||
async def rulings_for_paths(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
paths: list[str],
|
||||
*,
|
||||
seen: list[int] | set[int] | None = None,
|
||||
source: str,
|
||||
) -> dict:
|
||||
"""Rulings lines for the Systems whose files `paths` touch.
|
||||
|
||||
Returns {"lines": [...], "system_ids": [...]} — `system_ids` are the
|
||||
Systems shown IN FULL on this call, for the hook to add to the session's
|
||||
seen list and for the usage ledger; a System in `seen` gets a reference
|
||||
line and is in neither. A System with patterns but no rulings says
|
||||
nothing: there is no decision to deliver, and the charter already rides
|
||||
its records.
|
||||
"""
|
||||
out: dict = {"lines": [], "system_ids": []}
|
||||
if not project_id or not paths:
|
||||
return out
|
||||
try:
|
||||
already = {int(s) for s in (seen or [])}
|
||||
for system, hit in await systems_svc.systems_for_paths(user_id, project_id, paths):
|
||||
rulings = parse_rulings(system.description)
|
||||
if not rulings:
|
||||
continue
|
||||
if system.id in already:
|
||||
out["lines"].append(_reference_line(system, hit[0]))
|
||||
else:
|
||||
out["lines"].append(_full_line(system, hit[0], rulings))
|
||||
out["system_ids"].append(system.id)
|
||||
if out["system_ids"]:
|
||||
record_system_surfaced(
|
||||
user_id=user_id, system_ids=out["system_ids"], source=source,
|
||||
project_id=project_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("rulings arm failed", exc_info=True)
|
||||
return {"lines": [], "system_ids": []}
|
||||
return out
|
||||
@@ -0,0 +1,133 @@
|
||||
"""System usage telemetry — were an area's rulings read once shown?
|
||||
|
||||
The twin of `note_usage` and `rule_usage` for Systems. The
|
||||
rulings arm shows a System's rulings when a command or edit touches the
|
||||
System's files; a pull is somebody then opening the System (`get_system`). The
|
||||
ratio says whether delivering rulings by path earns its line.
|
||||
|
||||
Fire-and-forget like its siblings: telemetry never adds latency to, or
|
||||
breaks, the surface it observes, and failures report through the shared
|
||||
canary rather than vanishing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.base import iso
|
||||
from grimoire.models.system_usage import PULLED, SURFACED, SystemUsageEvent
|
||||
from grimoire.services.background import report_telemetry_failure, spawn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _insert_events(rows: list[dict]) -> None:
|
||||
"""Persist usage rows. Best-effort: failures degrade, visibly."""
|
||||
try:
|
||||
async with async_session() as session:
|
||||
session.add_all([SystemUsageEvent(**row) for row in rows])
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await report_telemetry_failure("system_usage", "write")
|
||||
|
||||
|
||||
def _rows(user_id, system_ids, event: str, source: str, project_id) -> list[dict]:
|
||||
pid = int(project_id or 0) or None
|
||||
return [
|
||||
{"user_id": user_id, "system_id": int(sid), "event": event,
|
||||
"source": source, "project_id": pid}
|
||||
for sid in system_ids
|
||||
]
|
||||
|
||||
|
||||
def record_system_surfaced(
|
||||
*, user_id: int | None, system_ids: list[int], source: str,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: these Systems' rulings were shown in full. A repeat
|
||||
rendered as a short reference is not a surfacing and is not recorded."""
|
||||
try:
|
||||
rows = _rows(user_id, system_ids, SURFACED, source, project_id)
|
||||
except Exception:
|
||||
logger.debug("system usage payload build failed", exc_info=True)
|
||||
return
|
||||
if rows:
|
||||
spawn(_insert_events(rows), site="system_usage_write")
|
||||
|
||||
|
||||
def record_system_pulled(
|
||||
*, user_id: int | None, system_id: int, source: str,
|
||||
project_id: int | None = None,
|
||||
) -> None:
|
||||
"""Fire-and-forget: a System was opened in full."""
|
||||
try:
|
||||
rows = _rows(user_id, [system_id], PULLED, source, project_id)
|
||||
except Exception:
|
||||
logger.debug("system usage payload build failed", exc_info=True)
|
||||
return
|
||||
spawn(_insert_events(rows), site="system_usage_write")
|
||||
|
||||
|
||||
def empty_system_usage() -> dict:
|
||||
"""The zero readout — a System no event has touched.
|
||||
|
||||
Rendered unconditionally, like `empty_rule_usage`, so a System predating
|
||||
the table reads as "never shown, never opened" rather than as a missing
|
||||
key. Every System in an install predates it, so for a while this is the
|
||||
normal state and must not look like a broken readout.
|
||||
|
||||
The same four keys as the client's `RecordUsage`, but read differently: a
|
||||
System's rulings are delivered IN FULL in the injected line, so a
|
||||
surfacing with no pull is the arm working, not dead weight. See
|
||||
`retrieval_telemetry`'s `system_usage` block for why no ratio is offered.
|
||||
"""
|
||||
return {
|
||||
"surfaced_count": 0,
|
||||
"pull_count": 0,
|
||||
"last_surfaced_at": None,
|
||||
"last_pulled_at": None,
|
||||
}
|
||||
|
||||
|
||||
async def usage_for_systems(system_ids: list[int]) -> dict[int, dict]:
|
||||
"""Aggregate usage for a set of Systems: {system_id: {counts + timestamps}}.
|
||||
|
||||
One GROUP BY for the whole list, never a query per row — this decorates a
|
||||
list view. Systems with no events come back as `empty_system_usage()`.
|
||||
Fails open: a readout that could break the Systems list is worse than a
|
||||
zero, but the failure is reported so the zero is not mistaken for silence.
|
||||
"""
|
||||
ids = [int(s) for s in system_ids]
|
||||
out: dict[int, dict] = {sid: empty_system_usage() for sid in ids}
|
||||
if not ids:
|
||||
return out
|
||||
try:
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
SystemUsageEvent.system_id,
|
||||
SystemUsageEvent.event,
|
||||
func.count().label("n"),
|
||||
func.max(SystemUsageEvent.created_at).label("last_at"),
|
||||
)
|
||||
.where(SystemUsageEvent.system_id.in_(ids))
|
||||
.group_by(SystemUsageEvent.system_id, SystemUsageEvent.event)
|
||||
)
|
||||
).all()
|
||||
except Exception:
|
||||
await report_telemetry_failure("system_usage", "readout")
|
||||
return out
|
||||
for system_id, event, n, last_at in rows:
|
||||
slot = out.get(int(system_id))
|
||||
if slot is None:
|
||||
continue
|
||||
if event == SURFACED:
|
||||
slot["surfaced_count"] = int(n)
|
||||
slot["last_surfaced_at"] = iso(last_at)
|
||||
elif event == PULLED:
|
||||
slot["pull_count"] = int(n)
|
||||
slot["last_pulled_at"] = iso(last_at)
|
||||
return out
|
||||
@@ -0,0 +1,537 @@
|
||||
"""System management + record<->system associations.
|
||||
|
||||
A System is a per-project, reusable, self-describing subsystem/area. Access is
|
||||
governed by the project's permission via services/access.py (multi-user ACL
|
||||
rule) — never a bare owner filter. Records (notes/tasks/issues) link to systems
|
||||
many-to-many through record_systems, mutable over time.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.system import RecordSystem, System
|
||||
from grimoire.services import access
|
||||
from grimoire.services import canonical_systems as canonical_systems_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def local_name_key(name: str) -> str:
|
||||
"""The within-project uniqueness key: case and spacing, nothing else.
|
||||
|
||||
Deliberately weaker than `canonical_slug`. This one answers "is this the
|
||||
same System I already have here", where the operator's own spelling is the
|
||||
thing being compared; the canonical slug answers "is this the same AREA as
|
||||
some other project's System", where spelling is exactly what must be
|
||||
ignored.
|
||||
"""
|
||||
return " ".join(name.split()).lower()
|
||||
|
||||
|
||||
# Bounds on what one System may claim. Generous for a real area — a handful of
|
||||
# directories and the odd stray file — and small enough that a pasted file
|
||||
# listing is refused rather than stored as the area's definition.
|
||||
MAX_PATH_PATTERNS = 50
|
||||
MAX_PATH_PATTERN_LENGTH = 300
|
||||
|
||||
|
||||
def normalize_repo_path(path: str) -> str:
|
||||
"""A path as the patterns see it: relative to the repo root, `/`-separated.
|
||||
|
||||
Shared by the patterns and the paths matched against them, so the two can
|
||||
never disagree about whether `./src/x.py` and `src/x.py` are one file.
|
||||
"""
|
||||
out = (path or "").strip().replace("\\", "/")
|
||||
while out.startswith("./"):
|
||||
out = out[2:]
|
||||
out = out.lstrip("/")
|
||||
while len(out) > 1 and out.endswith("/"):
|
||||
out = out[:-1]
|
||||
return out
|
||||
|
||||
|
||||
def normalize_path_patterns(patterns) -> list[str]:
|
||||
"""Validate and tidy a System's path patterns; ValueError says what is wrong.
|
||||
|
||||
Blank entries are dropped and repeats collapse, keeping the first-written
|
||||
order. A pattern that climbs out of the repo (`..`) is refused rather than
|
||||
dropped: it can never match, and storing it would read as coverage.
|
||||
"""
|
||||
if patterns is None:
|
||||
return []
|
||||
if isinstance(patterns, str) or not isinstance(patterns, (list, tuple)):
|
||||
raise ValueError("path_patterns must be a list of glob strings")
|
||||
out: list[str] = []
|
||||
for raw in patterns:
|
||||
if not isinstance(raw, str):
|
||||
raise ValueError("path_patterns must be a list of glob strings")
|
||||
pattern = normalize_repo_path(raw)
|
||||
if not pattern:
|
||||
continue
|
||||
if len(pattern) > MAX_PATH_PATTERN_LENGTH:
|
||||
raise ValueError(
|
||||
f"path pattern longer than {MAX_PATH_PATTERN_LENGTH} characters: "
|
||||
f"{pattern[:60]}…"
|
||||
)
|
||||
if ".." in pattern.split("/"):
|
||||
raise ValueError(
|
||||
f"path pattern {pattern!r} leaves the repo — patterns are "
|
||||
"relative to the repo root"
|
||||
)
|
||||
if pattern not in out:
|
||||
out.append(pattern)
|
||||
if len(out) > MAX_PATH_PATTERNS:
|
||||
raise ValueError(
|
||||
f"{len(out)} path patterns; a System takes at most "
|
||||
f"{MAX_PATH_PATTERNS} — name directories with `**` rather than "
|
||||
"listing their files"
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def path_matches(pattern: str, path: str) -> bool:
|
||||
"""Whether a repo-relative path falls under one pattern.
|
||||
|
||||
Glob semantics are `PurePosixPath.full_match`: `*` stays inside one path
|
||||
segment, `**` spans any number of them, and case counts. A pattern with
|
||||
no wildcard that names a directory covers everything under it, so
|
||||
`src/billing` means the directory the way a person writing it means it.
|
||||
"""
|
||||
path = normalize_repo_path(path)
|
||||
if not path or not pattern:
|
||||
return False
|
||||
candidate = PurePosixPath(path)
|
||||
return candidate.full_match(pattern) or candidate.full_match(f"{pattern}/**")
|
||||
|
||||
|
||||
def matching_patterns(patterns, path: str) -> list[str]:
|
||||
"""The patterns among `patterns` that `path` falls under, in their order."""
|
||||
return [p for p in (patterns or []) if path_matches(p, path)]
|
||||
|
||||
|
||||
async def assess_system_name(user_id: int, project_id: int, name: str) -> dict:
|
||||
"""What BOTH doors must know before minting a System name.
|
||||
|
||||
One service function, so the MCP tool and the web UI cannot answer the
|
||||
same question differently — a gate living in one door is a gate the other
|
||||
ships without.
|
||||
|
||||
Returns `{"duplicate": …|None, "canonical": …|None}`:
|
||||
|
||||
- `duplicate` — this project already has a System by that name. A hard stop
|
||||
for the caller: a second one splits the area's records across two piles.
|
||||
- `canonical` — the global catalog covers this area, with a `basis`.
|
||||
`exact` is mechanical and safe to apply on the spot; `overlap` is a
|
||||
judgment call and must be OFFERED, never applied. Neither ever blocks:
|
||||
an unmatched name is a project-specific area, which is legitimate.
|
||||
|
||||
Fails open on both arms — a naming aid must never break a create.
|
||||
"""
|
||||
out: dict = {"duplicate": None, "canonical": None}
|
||||
key = local_name_key(name)
|
||||
if not key:
|
||||
return out
|
||||
try:
|
||||
for existing in await list_systems(user_id, project_id, include_archived=True):
|
||||
if local_name_key(existing.name) == key:
|
||||
out["duplicate"] = {"id": existing.id, "name": existing.name}
|
||||
return out
|
||||
except Exception:
|
||||
logger.debug("system name assessment: local scan failed", exc_info=True)
|
||||
return out
|
||||
try:
|
||||
exact = await canonical_systems_svc.find_by_name(name)
|
||||
if exact is not None:
|
||||
out["canonical"] = {
|
||||
"id": exact.id, "name": exact.name, "basis": "exact",
|
||||
}
|
||||
return out
|
||||
# No exact hit: fall back to the same overlap scoring the review
|
||||
# surface uses, so a create-time offer and a later proposal never
|
||||
# disagree about which area a name resembles.
|
||||
near = await canonical_systems_svc.best_overlap(name)
|
||||
if near is not None:
|
||||
out["canonical"] = near
|
||||
except Exception:
|
||||
logger.debug("system name assessment: catalog lookup failed", exc_info=True)
|
||||
return out
|
||||
|
||||
|
||||
async def standard_systems() -> list[tuple[str, str]]:
|
||||
"""The standard cross-project vocabulary as (name, charter) pairs.
|
||||
|
||||
Reads the GLOBAL canonical catalog. The catalog is a table rather than a
|
||||
constant because a constant cannot be a foreign key: projects map their
|
||||
own Systems onto a catalog entry, so variant spellings of one area still
|
||||
resolve to it.
|
||||
"""
|
||||
return [(entry.name, entry.description or "") for entry in
|
||||
await canonical_systems_svc.list_canonical_systems()]
|
||||
|
||||
|
||||
async def seed_standard_systems(user_id: int, project_id: int) -> list[System]:
|
||||
"""Mint the standard starter set for a project that has NO Systems yet.
|
||||
Idempotent: a project with any System — the vocabulary
|
||||
already started, standard or not — gets nothing; the duplicate gate and
|
||||
the project's own judgment take it from there. [] without write access.
|
||||
|
||||
Seeded Systems are mapped to their catalog entry as they are created, so a
|
||||
project born this way needs no reconciliation pass later."""
|
||||
if await list_systems(user_id, project_id, include_archived=True):
|
||||
return []
|
||||
out: list[System] = []
|
||||
for index, entry in enumerate(await canonical_systems_svc.list_canonical_systems()):
|
||||
system = await create_system(
|
||||
user_id, project_id, entry.name, description=entry.description,
|
||||
order_index=index, canonical_id=entry.id,
|
||||
)
|
||||
if system is None:
|
||||
break
|
||||
out.append(system)
|
||||
return out
|
||||
|
||||
|
||||
def embed_system(system) -> None:
|
||||
"""Refresh a System's charter vectors, fire-and-forget.
|
||||
|
||||
The twin of `notes.embed_note`, and here for the same reason: at
|
||||
the service, so every door gets it by construction rather than each route
|
||||
and tool remembering. A charter edited through one door and not another
|
||||
would stay findable by what it used to say, and nothing would report it.
|
||||
|
||||
Not called on delete. `delete_system` is a SOFT delete and the search joins
|
||||
through `System`, so a deleted System's vectors are already unreachable —
|
||||
and leaving them means a restore is findable again immediately instead of
|
||||
waiting for the next startup backfill.
|
||||
|
||||
Import is lazy so importing this module doesn't pull in the embedding
|
||||
model; a missing event loop (unit tests, scripts) is ordinary, not an
|
||||
error; exceptions are swallowed because a System that saved must not fail
|
||||
on its index refresh.
|
||||
"""
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from grimoire.services.embeddings import upsert_system_embedding
|
||||
|
||||
asyncio.create_task(
|
||||
upsert_system_embedding(system.id, system.name, system.description)
|
||||
)
|
||||
except RuntimeError:
|
||||
pass # no running loop — a sync caller, not a failure
|
||||
except Exception: # noqa: BLE001 - never let indexing break a write
|
||||
logger.exception("embedding refresh failed for system %s", system.id)
|
||||
|
||||
|
||||
async def create_system(
|
||||
user_id: int,
|
||||
project_id: int,
|
||||
name: str,
|
||||
description: str | None = None,
|
||||
color: str | None = None,
|
||||
order_index: int = 0,
|
||||
canonical_id: int | None = None,
|
||||
path_patterns: list[str] | None = None,
|
||||
) -> System | None:
|
||||
"""Create a System. None if the user can't write the project.
|
||||
|
||||
`canonical_id` maps the new System onto the global catalog; leaving it None
|
||||
is fine — an unmapped System is fully usable, and the mapping can be
|
||||
proposed later (services/canonical_systems.propose_mappings).
|
||||
|
||||
`path_patterns` are validated here, so every door refuses the same bad
|
||||
pattern with the same message (ValueError).
|
||||
"""
|
||||
patterns = normalize_path_patterns(path_patterns)
|
||||
if not await access.can_write_project(user_id, project_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
system = System(
|
||||
user_id=user_id,
|
||||
project_id=project_id,
|
||||
name=name.strip(),
|
||||
description=description,
|
||||
color=color,
|
||||
order_index=order_index,
|
||||
canonical_id=canonical_id,
|
||||
path_patterns=patterns,
|
||||
)
|
||||
session.add(system)
|
||||
await session.commit()
|
||||
await session.refresh(system)
|
||||
embed_system(system)
|
||||
return system
|
||||
|
||||
|
||||
async def get_system(user_id: int, system_id: int) -> System | None:
|
||||
"""Fetch a System if the user can read its project."""
|
||||
async with async_session() as session:
|
||||
system = await session.get(System, system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
if not await access.can_read_project(user_id, system.project_id):
|
||||
return None
|
||||
return system
|
||||
|
||||
|
||||
async def list_systems(
|
||||
user_id: int, project_id: int, include_archived: bool = False
|
||||
) -> list[System]:
|
||||
"""A project's systems (active by default). [] if no read access."""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
query = select(System).where(
|
||||
System.project_id == project_id,
|
||||
System.deleted_at.is_(None),
|
||||
)
|
||||
if not include_archived:
|
||||
query = query.where(System.status == "active")
|
||||
query = query.order_by(System.order_index.asc(), System.created_at.asc())
|
||||
result = await session.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def systems_for_paths(
|
||||
user_id: int, project_id: int, paths: list[str],
|
||||
) -> list[tuple[System, list[str]]]:
|
||||
"""The project's active Systems whose patterns cover any of `paths`.
|
||||
|
||||
Each comes with the paths it matched, in the order given. A path can match
|
||||
several Systems — areas overlap, and every one it belongs to answers for
|
||||
it — so nothing here picks a winner. Systems with no patterns never match:
|
||||
an area that has not named its files is not claiming all of them.
|
||||
"""
|
||||
wanted = [p for p in (normalize_repo_path(x) for x in paths or []) if p]
|
||||
if not wanted:
|
||||
return []
|
||||
out: list[tuple[System, list[str]]] = []
|
||||
for system in await list_systems(user_id, project_id):
|
||||
patterns = system.path_patterns or []
|
||||
if not patterns:
|
||||
continue
|
||||
hit = [p for p in wanted if matching_patterns(patterns, p)]
|
||||
if hit:
|
||||
out.append((system, hit))
|
||||
return out
|
||||
|
||||
async def update_system(user_id: int, system_id: int, **fields: object) -> System | None:
|
||||
"""Update a System if the user can write its project."""
|
||||
# canonical_id is deliberately NOT settable here: canonical_systems.
|
||||
# set_system_canonical is its single writer, because it also validates the
|
||||
# catalog entry is live. Two entry points onto one column is the drift this
|
||||
# table exists to end.
|
||||
allowed = {"name", "description", "color", "status", "order_index", "path_patterns"}
|
||||
# Validated before anything is read, so a refused pattern changes nothing.
|
||||
# `[]` is a value here, not "leave unchanged": it clears the paths.
|
||||
if fields.get("path_patterns") is not None:
|
||||
fields["path_patterns"] = normalize_path_patterns(fields["path_patterns"])
|
||||
async with async_session() as session:
|
||||
system = await session.get(System, system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return None
|
||||
if not await access.can_write_project(user_id, system.project_id):
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if key in allowed and value is not None:
|
||||
setattr(system, key, value)
|
||||
system.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(system)
|
||||
embed_system(system)
|
||||
return system
|
||||
|
||||
|
||||
async def archive_system(user_id: int, system_id: int) -> System | None:
|
||||
return await update_system(user_id, system_id, status="archived")
|
||||
|
||||
|
||||
async def delete_system(user_id: int, system_id: int) -> bool:
|
||||
"""Soft-delete a System (recoverable). Requires project write access."""
|
||||
async with async_session() as session:
|
||||
system = await session.get(System, system_id)
|
||||
if system is None or system.deleted_at is not None:
|
||||
return False
|
||||
if not await access.can_write_project(user_id, system.project_id):
|
||||
return False
|
||||
system.deleted_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
# --- record <-> system associations (many-to-many, mutable) ---
|
||||
|
||||
async def set_record_systems(
|
||||
user_id: int, note_id: int, system_ids: list[int]
|
||||
) -> list[int] | None:
|
||||
"""Replace a record's system associations with `system_ids` (set semantics).
|
||||
|
||||
Returns the resulting associated system ids, or None if the user can't write
|
||||
the record. Silently drops ids that don't exist or the user can't read —
|
||||
associations only link accessible systems.
|
||||
"""
|
||||
if not await access.can_write_note(user_id, note_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
wanted: list[int] = []
|
||||
for sid in dict.fromkeys(system_ids): # de-dup, preserve order
|
||||
system = await session.get(System, sid)
|
||||
if system is None or system.deleted_at is not None:
|
||||
continue
|
||||
if not await access.can_read_project(user_id, system.project_id):
|
||||
continue
|
||||
wanted.append(sid)
|
||||
|
||||
existing = set(
|
||||
(
|
||||
await session.execute(
|
||||
select(RecordSystem.system_id).where(RecordSystem.note_id == note_id)
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
wanted_set = set(wanted)
|
||||
|
||||
to_remove = existing - wanted_set
|
||||
if to_remove:
|
||||
await session.execute(
|
||||
delete(RecordSystem).where(
|
||||
RecordSystem.note_id == note_id,
|
||||
RecordSystem.system_id.in_(to_remove),
|
||||
)
|
||||
)
|
||||
for sid in wanted:
|
||||
if sid not in existing:
|
||||
session.add(RecordSystem(note_id=note_id, system_id=sid))
|
||||
await session.commit()
|
||||
return wanted
|
||||
|
||||
|
||||
async def list_record_systems(user_id: int, note_id: int) -> list[System]:
|
||||
"""Systems associated with a record (if the user can read it)."""
|
||||
if not await access.can_read_note(user_id, note_id):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(System)
|
||||
.join(RecordSystem, RecordSystem.system_id == System.id)
|
||||
.where(RecordSystem.note_id == note_id, System.deleted_at.is_(None))
|
||||
.order_by(System.order_index.asc(), System.name.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def system_names_for(note_ids: set[int]) -> dict[int, list[str]]:
|
||||
"""{note_id: [system name, …]} in one query, for records ALREADY read.
|
||||
|
||||
For decorating a result set the caller was allowed to see — an injected
|
||||
menu line says which part of the project a record is about, so the reader
|
||||
can place it without opening it. No access check here for that
|
||||
reason: the ids come from a search that applied one, and a system name is
|
||||
metadata of the record, not a record of its own.
|
||||
|
||||
Fails soft, like `access.owner_names_for`: a menu without its system labels
|
||||
is a cosmetic downgrade, and failing the whole injection over one is not.
|
||||
"""
|
||||
if not note_ids:
|
||||
return {}
|
||||
try:
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(RecordSystem.note_id, System.name)
|
||||
.join(System, System.id == RecordSystem.system_id)
|
||||
.where(RecordSystem.note_id.in_(note_ids), System.deleted_at.is_(None))
|
||||
.order_by(System.order_index.asc(), System.name.asc())
|
||||
)
|
||||
).all()
|
||||
except Exception:
|
||||
logger.warning("System-name lookup failed; menu lines go unlabelled", exc_info=True)
|
||||
return {}
|
||||
out: dict[int, list[str]] = {}
|
||||
for note_id, name in rows:
|
||||
out.setdefault(int(note_id), []).append(name)
|
||||
return out
|
||||
|
||||
|
||||
async def list_records_for_system(
|
||||
user_id: int, system_id: int, kind: str | None = None, open_only: bool = False
|
||||
) -> list[Note]:
|
||||
"""Records associated with a System. `kind` filters task_kind (e.g. 'issue');
|
||||
`open_only` limits to tasks not done/cancelled. [] if no read access."""
|
||||
system = await get_system(user_id, system_id)
|
||||
if system is None:
|
||||
return []
|
||||
async with async_session() as session:
|
||||
query = (
|
||||
select(Note)
|
||||
.join(RecordSystem, RecordSystem.note_id == Note.id)
|
||||
.where(RecordSystem.system_id == system_id, Note.deleted_at.is_(None))
|
||||
)
|
||||
if kind:
|
||||
query = query.where(Note.task_kind == kind)
|
||||
if open_only:
|
||||
query = query.where(Note.status.not_in(["done", "cancelled"]))
|
||||
query = query.order_by(Note.updated_at.desc())
|
||||
result = await session.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def count_open_issues(user_id: int, project_id: int) -> int:
|
||||
"""Count open (not done/cancelled) issues in a project. 0 if no read access."""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return 0
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(func.count(Note.id)).where(
|
||||
Note.project_id == project_id,
|
||||
Note.task_kind == "issue",
|
||||
Note.status.isnot(None),
|
||||
Note.status.not_in(["done", "cancelled"]),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
|
||||
async def open_issue_counts_by_system(user_id: int, project_id: int) -> dict[int, int]:
|
||||
"""{system_id: open-issue count} for all systems in a project, in one query.
|
||||
{} if no read access."""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return {}
|
||||
async with async_session() as session:
|
||||
rows = await session.execute(
|
||||
select(RecordSystem.system_id, func.count(Note.id))
|
||||
.join(Note, Note.id == RecordSystem.note_id)
|
||||
.join(System, System.id == RecordSystem.system_id)
|
||||
.where(
|
||||
System.project_id == project_id,
|
||||
Note.task_kind == "issue",
|
||||
Note.status.isnot(None),
|
||||
Note.status.not_in(["done", "cancelled"]),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
.group_by(RecordSystem.system_id)
|
||||
)
|
||||
return {sid: cnt for sid, cnt in rows.fetchall()}
|
||||
|
||||
|
||||
async def list_issues(user_id: int, project_id: int, open_only: bool = True) -> list[Note]:
|
||||
"""Issues in a project (open by default), newest first. [] if no read access."""
|
||||
if not await access.can_read_project(user_id, project_id):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
query = select(Note).where(
|
||||
Note.project_id == project_id,
|
||||
Note.task_kind == "issue",
|
||||
Note.status.isnot(None),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
if open_only:
|
||||
query = query.where(Note.status.not_in(["done", "cancelled"]))
|
||||
query = query.order_by(Note.updated_at.desc())
|
||||
result = await session.execute(query)
|
||||
return list(result.scalars().all())
|
||||
@@ -0,0 +1,319 @@
|
||||
"""A task's claim — which session is working it right now.
|
||||
|
||||
`status` says where THE WORK stands and is durable: `in_progress` means
|
||||
committed to, not finished. It cannot also say "someone is on this now",
|
||||
because nothing ever clears it — a session that crashes, is killed or simply
|
||||
moves on leaves `in_progress` behind, and the row goes on asserting attention
|
||||
nobody is paying. The claim is the other half, a property of a session's
|
||||
attention rather than of the work, and it is built so that nothing has to
|
||||
clear it either.
|
||||
|
||||
WHO STAMPS IT. The server, on the write that IS the work: a transition to
|
||||
`in_progress` and a work log on an open task. No tool asks the model to claim
|
||||
anything, because a claim the model has to remember is the flag this replaces.
|
||||
Any MCP client gets the lease; the Claude Code plugin's PostToolUse hook then
|
||||
binds the harness's session id to it (`bind_session`), so the claim can say
|
||||
WHICH session and not only "someone, recently" — the harness reports the id,
|
||||
the model asserts nothing.
|
||||
|
||||
HOW IT DIES. On read. A claim is live while its last touch is inside
|
||||
`CLAIM_LEASE`; past that it reads as dead, whatever the row still holds. There
|
||||
is no sweep: a job that tidies claims would reintroduce exactly the dependency
|
||||
on something running that this is designed out of. A status that ends or
|
||||
un-starts the work (done, cancelled, todo) releases it outright.
|
||||
|
||||
`in_progress` with no live claim is the state this exists to make sayable:
|
||||
committed to, and nobody on it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from grimoire.models.note import Note
|
||||
|
||||
# How long a claim stays live after its last touch. Long enough that a
|
||||
# compaction, a resume or a long read does not kill it; short enough that a
|
||||
# session gone overnight reads as gone. The cost either way is stated rather
|
||||
# than hidden: readers show the age beside `live`, never the boolean alone.
|
||||
#
|
||||
# NOT A SETTING, deliberately. Settings are per user, and a
|
||||
# claim is read by everyone who can see the task: a per-user lease would make
|
||||
# one shared task read as live to one collaborator and dead to another, and
|
||||
# "is anyone on this?" only means something if every reader gets the same
|
||||
# answer. It is also read synchronously in `to_dict`, which has no user to ask.
|
||||
CLAIM_LEASE = timedelta(hours=2)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def claim_is_live(note: Note, now: datetime | None = None) -> bool:
|
||||
touched = note.claim_touched_at
|
||||
return touched is not None and (now or _now()) - touched < CLAIM_LEASE
|
||||
|
||||
|
||||
def stamp_claim(note: Note, user_id: int, now: datetime | None = None) -> None:
|
||||
"""Record that `user_id` is working this task now.
|
||||
|
||||
The most recent worker holds it: a live claim by someone else is taken
|
||||
over rather than refused, because the write that stamps a claim has already
|
||||
happened — it is the evidence of who is on the task, and refusing the claim
|
||||
would only make the record less true. `claimed_at` restarts whenever the
|
||||
holder changes or the previous claim had died, so "since" means since THIS
|
||||
stretch of attention, not since the task was first touched.
|
||||
"""
|
||||
now = now or _now()
|
||||
if note.claimed_by != user_id or not claim_is_live(note, now):
|
||||
note.claimed_by = user_id
|
||||
note.claimed_at = now
|
||||
note.claim_session = None
|
||||
note.claim_touched_at = now
|
||||
|
||||
|
||||
def release_claim(note: Note) -> None:
|
||||
"""Clear the claim. Idempotent — releasing nothing is not an error."""
|
||||
note.claimed_by = None
|
||||
note.claimed_at = None
|
||||
note.claim_touched_at = None
|
||||
note.claim_session = None
|
||||
|
||||
|
||||
def claim_state(note: Note, now: datetime | None = None) -> dict | None:
|
||||
"""The claim as a reader sees it, or None when there has never been one.
|
||||
|
||||
A dead claim is returned, not hidden: "last worked by that session three
|
||||
days ago" is what a resuming session needs to know, and `live: false` says
|
||||
no-one should read it as current.
|
||||
"""
|
||||
if note.claimed_at is None:
|
||||
return None
|
||||
from grimoire.models.base import iso
|
||||
|
||||
return {
|
||||
"held_by": note.claimed_by,
|
||||
"session": note.claim_session,
|
||||
"since": iso(note.claimed_at),
|
||||
"touched": iso(note.claim_touched_at),
|
||||
"live": claim_is_live(note, now),
|
||||
}
|
||||
|
||||
|
||||
async def bind_session(user_id: int, task_id: int, session_id: str) -> dict | None:
|
||||
"""Attach a harness-reported session id to the caller's live claim.
|
||||
|
||||
Called by the plugin's PostToolUse hook after `update_task` or
|
||||
`add_task_log`. A no-op — returning None — when the task is not writable
|
||||
by the caller, has no live claim, or the live claim is someone else's: the
|
||||
hook reports what happened, it cannot create a claim the server did not
|
||||
stamp. A different session taking over a live claim restarts `since`,
|
||||
for the reason `stamp_claim` gives.
|
||||
"""
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.services.access import can_write_note
|
||||
|
||||
session_id = (session_id or "").strip()[:200]
|
||||
if not session_id or not await can_write_note(user_id, task_id):
|
||||
return None
|
||||
async with async_session() as session:
|
||||
note = await session.get(Note, task_id)
|
||||
if note is None or note.claimed_by != user_id or not claim_is_live(note):
|
||||
return None
|
||||
now = _now()
|
||||
if note.claim_session not in (None, session_id):
|
||||
note.claimed_at = now
|
||||
note.claim_session = session_id
|
||||
note.claim_touched_at = now
|
||||
await session.commit()
|
||||
return claim_state(note, now)
|
||||
|
||||
|
||||
# --- The readers --------------------------------------------------------------
|
||||
#
|
||||
# A claim nobody reads is worth nothing. SessionStart is the
|
||||
# reader that pays rent to the session that set it, and it branches on the
|
||||
# `source` the host sends, because the same claim means different things
|
||||
# depending on what just happened to the context:
|
||||
#
|
||||
# compact the context was summarised away and the claim is certainly ours.
|
||||
# Push the claimed work AND its latest log entries — the state a
|
||||
# compaction destroys, which the record already holds. A count of
|
||||
# open tasks cannot answer "where were we".
|
||||
# clear the context was wiped, so the same push; and other sessions' claims
|
||||
# are worth knowing about, as on a startup.
|
||||
# startup a new session. Claims held by OTHER sessions are the news: live
|
||||
# ones may be running right now, dead ones were abandoned mid-task.
|
||||
# fork the session carries a conversation that held claims under another
|
||||
# id. Two sessions now believe they hold the same work, so the
|
||||
# live claims are named as possibly-the-parent's, with what a write
|
||||
# does about it.
|
||||
# resume the context was restored intact. Say nothing.
|
||||
|
||||
# What a session is told, per source. Pure data so the branch is one lookup.
|
||||
_PUSH_OWN = {"compact", "clear"}
|
||||
_NAME_OTHERS = {"startup", "clear", "fork"}
|
||||
|
||||
# Caps: a session-start block is read by every session, so it is sized for the
|
||||
# few claims that matter rather than for the worst case.
|
||||
_OWN_CAP = 5
|
||||
_OTHERS_CAP = 5
|
||||
_LOGS_PER_TASK = 2
|
||||
_LOG_CHARS = 600
|
||||
|
||||
|
||||
def _age(when: datetime | None, now: datetime) -> str:
|
||||
if when is None:
|
||||
return "at an unknown time"
|
||||
secs = max(0, int((now - when).total_seconds()))
|
||||
if secs < 90:
|
||||
return "just now"
|
||||
if secs < 5400:
|
||||
return f"{secs // 60}m ago"
|
||||
if secs < 2 * 86400:
|
||||
return f"{secs // 3600}h ago"
|
||||
return f"{secs // 86400}d ago"
|
||||
|
||||
|
||||
def render_claims(
|
||||
source: str,
|
||||
session_id: str,
|
||||
claims: list,
|
||||
logs: dict[int, list],
|
||||
now: datetime | None = None,
|
||||
) -> list[str]:
|
||||
"""The claim section of the SessionStart context, as markdown lines.
|
||||
|
||||
`claims` are the caller's claimed tasks (objects with id, title, status and
|
||||
the claim columns); `logs` maps a task id to its newest log entries
|
||||
(objects with `created_at` and `content`), newest first. Empty when there is
|
||||
nothing this source should say — silence is the right answer on a resume,
|
||||
and on any start with no claims.
|
||||
"""
|
||||
from grimoire.services.text import elide
|
||||
|
||||
now = now or _now()
|
||||
source = (source or "").strip()
|
||||
session_id = (session_id or "").strip()
|
||||
ours = [c for c in claims if claim_is_live(c, now)
|
||||
and (c.claim_session == session_id or c.claim_session is None)]
|
||||
others_live = [c for c in claims if claim_is_live(c, now)
|
||||
and c.claim_session not in (None, session_id)]
|
||||
abandoned = [c for c in claims if not claim_is_live(c, now)
|
||||
and c.status == "in_progress"]
|
||||
|
||||
lines: list[str] = []
|
||||
if source in _PUSH_OWN and session_id and ours:
|
||||
lines += [
|
||||
"",
|
||||
"## In flight — the work this session had claimed",
|
||||
"Grimoire's record of what you were doing before the context was "
|
||||
"lost. Carry on from here; the full log is `get_task(id)`.",
|
||||
]
|
||||
for c in ours[:_OWN_CAP]:
|
||||
lines.append(
|
||||
f"- #{c.id} \"{c.title}\" ({c.status}) — claimed "
|
||||
f"{_age(c.claimed_at, now)}, last touched {_age(c.claim_touched_at, now)}"
|
||||
)
|
||||
for entry in logs.get(c.id, [])[:_LOGS_PER_TASK]:
|
||||
text, _ = elide(" ".join((entry.content or "").split()), _LOG_CHARS)
|
||||
lines.append(f" - log {_age(entry.created_at, now)}: {text}")
|
||||
if source in _NAME_OTHERS and (others_live or abandoned):
|
||||
lines += ["", "## Work other sessions were doing"]
|
||||
if source == "fork":
|
||||
lines.append(
|
||||
"This session was forked, so a live claim below may be the "
|
||||
"session you were forked from — two sessions now think they "
|
||||
"hold it. Your next log or status change on a task moves its "
|
||||
"claim here; leave it alone if the other session is still on it."
|
||||
)
|
||||
for c in others_live[:_OTHERS_CAP]:
|
||||
lines.append(
|
||||
f"- #{c.id} \"{c.title}\" — claimed by another session, last "
|
||||
f"touched {_age(c.claim_touched_at, now)}. It may still be "
|
||||
"running; check before working the same task."
|
||||
)
|
||||
for c in abandoned[:_OTHERS_CAP]:
|
||||
lines.append(
|
||||
f"- #{c.id} \"{c.title}\" — in progress, but the session "
|
||||
f"working it went quiet {_age(c.claim_touched_at, now)} without "
|
||||
"finishing. Read its log and continue it, or set it back to todo."
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
async def claims_for_session_start(
|
||||
user_id: int, project_id: int, source: str, session_id: str,
|
||||
) -> list[str]:
|
||||
"""Load the caller's claims (in the active project, when one resolved) and
|
||||
their newest logs, and render them for this `source`.
|
||||
|
||||
"The caller's claims" is `claimed_by == user_id` — a statement about whose
|
||||
attention a claim records, not an access filter: a claim is only ever
|
||||
stamped by a write the caller was already allowed to make.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.task_log import TaskLog
|
||||
|
||||
# No source means the caller did not ask — another client, or an adapter
|
||||
# older than this section — and a resume restored everything already.
|
||||
if (source or "") in ("", "resume"):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
q = select(Note).where(
|
||||
Note.claimed_by == user_id,
|
||||
Note.claimed_at.is_not(None),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
if project_id:
|
||||
q = q.where(Note.project_id == project_id)
|
||||
claims = list((await session.execute(
|
||||
q.order_by(Note.claim_touched_at.desc()).limit(_OWN_CAP + 2 * _OTHERS_CAP)
|
||||
)).scalars().all())
|
||||
logs: dict[int, list] = {}
|
||||
if claims:
|
||||
rows = (await session.execute(
|
||||
select(TaskLog)
|
||||
.where(TaskLog.task_id.in_([c.id for c in claims]))
|
||||
.order_by(TaskLog.created_at.desc())
|
||||
)).scalars().all()
|
||||
for row in rows:
|
||||
bucket = logs.setdefault(row.task_id, [])
|
||||
if len(bucket) < _LOGS_PER_TASK:
|
||||
bucket.append(row)
|
||||
return render_claims(source, session_id, claims, logs)
|
||||
|
||||
|
||||
async def release_session(user_id: int, session_id: str) -> int:
|
||||
"""Release every claim the caller holds under `session_id`.
|
||||
|
||||
Called by the plugin's SessionEnd hook: the session's context is about to
|
||||
stop existing, so the attention its claims record is ending too. A TIDY-UP,
|
||||
not the guarantee — SessionEnd does not fire on a crash, a killed terminal
|
||||
or a dropped connection, and those are what the lease is for. Returns how
|
||||
many were released; 0 is the ordinary answer for a session that claimed
|
||||
nothing.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
|
||||
session_id = (session_id or "").strip()[:200]
|
||||
if not session_id:
|
||||
return 0
|
||||
async with async_session() as session:
|
||||
held = (await session.execute(
|
||||
select(Note).where(
|
||||
Note.claimed_by == user_id, Note.claim_session == session_id,
|
||||
)
|
||||
)).scalars().all()
|
||||
for note in held:
|
||||
release_claim(note)
|
||||
await session.commit()
|
||||
return len(held)
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Task work log service."""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.task_log import TaskLog
|
||||
from grimoire.models.note import Note, TaskStatus
|
||||
from grimoire.services.access import can_read_note, can_write_note, readable_notes_clause
|
||||
from grimoire.services.task_claims import stamp_claim
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
async def _refresh_task_document(session, task_id: int) -> None:
|
||||
"""Re-embed the task whose work log just changed.
|
||||
|
||||
A task's embedded document carries its logs, so a log written and not
|
||||
indexed is half a surface: the entry is readable and unfindable, and the next session rebuilds what this one ruled
|
||||
out. Create, edit and delete all go through here — an edited log that keeps
|
||||
matching its old wording is the stale-vector problem `upsert_note_embedding`
|
||||
already refuses to leave behind for a body.
|
||||
|
||||
Loads the NOTE rather than synthesising one. `embed_note` reads
|
||||
`title`, `body` and the OWNER's `user_id` off what it is handed, so a
|
||||
stand-in row would index the logs under an empty title — throwing away the
|
||||
per-chunk topical anchor that makes any of this discriminative — and file
|
||||
the vectors under the wrong user.
|
||||
|
||||
Failure is swallowed the way `embed_note`'s own is: a log that saved must
|
||||
not fail on its index refresh. The startup backfill is the backstop.
|
||||
"""
|
||||
from grimoire.services.notes import embed_note
|
||||
|
||||
try:
|
||||
result = await session.execute(select(Note).where(Note.id == task_id))
|
||||
note = result.scalars().first()
|
||||
if note is not None:
|
||||
embed_note(note)
|
||||
except Exception: # noqa: BLE001 - indexing never breaks a write
|
||||
logger.exception("embedding refresh failed for task %s", task_id)
|
||||
|
||||
|
||||
async def create_log(
|
||||
user_id: int,
|
||||
task_id: int,
|
||||
content: str,
|
||||
duration_minutes: int | None = None,
|
||||
) -> TaskLog:
|
||||
# Whoever may WRITE the task may log on it — a collaborator on a shared
|
||||
# project included. An owner-only filter would refuse exactly the person a
|
||||
# shared task exists for.
|
||||
if not await can_write_note(user_id, task_id):
|
||||
raise ValueError(f"Task {task_id} not found")
|
||||
async with async_session() as session:
|
||||
result = await session.execute(select(Note).where(Note.id == task_id))
|
||||
task = result.scalars().first()
|
||||
if task is None:
|
||||
raise ValueError(f"Task {task_id} not found")
|
||||
# Logging IS working the task, so it stamps the claim —
|
||||
# unless the work is over: a retrospective note on a closed task is not
|
||||
# a session picking it up.
|
||||
if task.status not in (TaskStatus.done.value, TaskStatus.cancelled.value):
|
||||
stamp_claim(task, user_id)
|
||||
log = TaskLog(
|
||||
task_id=task_id,
|
||||
user_id=user_id,
|
||||
content=content,
|
||||
duration_minutes=duration_minutes,
|
||||
)
|
||||
session.add(log)
|
||||
await session.commit()
|
||||
await session.refresh(log)
|
||||
await _refresh_task_document(session, task_id)
|
||||
return log
|
||||
|
||||
|
||||
async def list_logs(user_id: int, task_id: int) -> list[TaskLog]:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(TaskLog)
|
||||
.where(TaskLog.task_id == task_id, TaskLog.user_id == user_id)
|
||||
.order_by(TaskLog.created_at.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def logs_for_task(
|
||||
user_id: int, task_id: int, limit: int = 0
|
||||
) -> list[TaskLog]:
|
||||
"""Every work log on a task, NEWEST FIRST, whoever wrote it.
|
||||
|
||||
Two differences from `list_logs`, both deliberate:
|
||||
|
||||
1. Scoped by who may read the TASK, not by who wrote each entry. A work
|
||||
log belongs to the task, and on a shared task the owner's record is
|
||||
exactly what a collaborator needs — `list_logs`' `TaskLog.user_id ==
|
||||
user_id` returns them an empty list, which reads as "no work has been
|
||||
done" rather than "not yours". The permission question is asked of the
|
||||
note through `can_read_note`, so this is not an unscoped
|
||||
read with the check left to the caller.
|
||||
2. Newest first, because the question a reader asks of a work log is
|
||||
"where does this actually stand", and the answer is the last entry.
|
||||
`list_logs` stays ascending — the web UI renders a narrative.
|
||||
|
||||
`limit` of 0 means all of them. An unreadable or missing task returns []
|
||||
the same way an empty one does: a work log is not a channel for proving a
|
||||
record exists.
|
||||
"""
|
||||
if not await can_read_note(user_id, task_id):
|
||||
return []
|
||||
async with async_session() as session:
|
||||
stmt = (
|
||||
select(TaskLog)
|
||||
.where(TaskLog.task_id == task_id)
|
||||
.order_by(TaskLog.created_at.desc(), TaskLog.id.desc())
|
||||
)
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
result = await session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def count_logs_for_task(user_id: int, task_id: int) -> int:
|
||||
"""How many work logs a task carries. Same scoping as logs_for_task."""
|
||||
if not await can_read_note(user_id, task_id):
|
||||
return 0
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(func.count(TaskLog.id)).where(TaskLog.task_id == task_id)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
|
||||
async def log_counts_for_tasks(
|
||||
user_id: int, task_ids: list[int]
|
||||
) -> dict[int, int]:
|
||||
"""Log counts for a whole page of tasks in ONE query.
|
||||
|
||||
A per-row lookup would be N+1 by construction — and so would a per-row
|
||||
`can_read_note` — which is the reason a list surface would otherwise keep
|
||||
omitting this and leave a reader to guess which rows carry a record. So
|
||||
the permission is expressed as set membership with
|
||||
`readable_notes_clause` and folded into the same statement, the pattern
|
||||
that function exists for.
|
||||
|
||||
Tasks with no logs are absent from the mapping; callers zero-fill, so
|
||||
"none" reads as a count rather than a missing key.
|
||||
"""
|
||||
ids = [int(t) for t in task_ids if t]
|
||||
if not ids:
|
||||
return {}
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(TaskLog.task_id, func.count(TaskLog.id))
|
||||
.join(Note, Note.id == TaskLog.task_id)
|
||||
.where(TaskLog.task_id.in_(ids), readable_notes_clause(user_id))
|
||||
.group_by(TaskLog.task_id)
|
||||
)
|
||||
return {int(tid): int(n) for tid, n in result.all()}
|
||||
|
||||
|
||||
async def update_log(
|
||||
user_id: int,
|
||||
log_id: int,
|
||||
content: str | None = None,
|
||||
duration_minutes: object = _UNSET,
|
||||
) -> TaskLog | None:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(TaskLog).where(TaskLog.id == log_id, TaskLog.user_id == user_id)
|
||||
)
|
||||
log = result.scalars().first()
|
||||
if log is None:
|
||||
return None
|
||||
if content is not None:
|
||||
log.content = content
|
||||
if duration_minutes is not _UNSET:
|
||||
log.duration_minutes = duration_minutes # type: ignore[assignment]
|
||||
log.updated_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
await session.refresh(log)
|
||||
await _refresh_task_document(session, log.task_id)
|
||||
return log
|
||||
|
||||
|
||||
async def delete_log(user_id: int, log_id: int) -> bool:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(TaskLog).where(TaskLog.id == log_id, TaskLog.user_id == user_id)
|
||||
)
|
||||
log = result.scalars().first()
|
||||
if log is None:
|
||||
return False
|
||||
task_id = log.task_id
|
||||
await session.delete(log)
|
||||
await session.commit()
|
||||
await _refresh_task_document(session, task_id)
|
||||
return True
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Text shortening for surfaces that cannot show a record whole.
|
||||
|
||||
One function, in one place, because both doors shorten and two copies would
|
||||
drift — and because the reasoning below is the part that matters and should
|
||||
not have to be re-derived at each call site.
|
||||
"""
|
||||
|
||||
|
||||
def elide(text: str, budget: int) -> tuple[str, bool]:
|
||||
"""Cut to `budget` characters from the MIDDLE, keeping both ends.
|
||||
|
||||
A head-only cut — `text[:800]` — decides what a reader sees by character
|
||||
position, which is uncorrelated with what matters. Prose does not put its
|
||||
conclusion first: a passage that opens with what was attempted and closes
|
||||
with "so this shipped in 04775c3" loses exactly the sentence that answers
|
||||
the question. Worse, the reader cannot tell: a truncation marker says that
|
||||
something was removed, never whether it mattered, so the decision "should
|
||||
I look deeper?" gets made on evidence selected by length.
|
||||
|
||||
So keep the opening (what this is about) AND the closing (where it landed),
|
||||
and state in between how much went. Two thirds to the head because that is
|
||||
where the subject is established; a conclusion needs less room to carry.
|
||||
|
||||
Returns (text, was_cut). A `budget` of 0 or less means no cut.
|
||||
|
||||
This is the fallback, not the goal. Where the system knows WHICH span of a
|
||||
record is the relevant one — a semantic search knows exactly that, and
|
||||
stores it — show that span and say so. Reach for this only when
|
||||
nothing identifies a better part than "all of it".
|
||||
"""
|
||||
if budget <= 0 or len(text) <= budget:
|
||||
return text, False
|
||||
head_len = max(1, budget * 2 // 3)
|
||||
tail_len = max(1, budget - head_len)
|
||||
omitted = len(text) - head_len - tail_len
|
||||
head = text[:head_len].rstrip()
|
||||
tail = text[-tail_len:].lstrip()
|
||||
return f"{head}\n\n[… {omitted} characters omitted …]\n\n{tail}", True
|
||||
|
||||
|
||||
# What a search result shows a reader, as one decision made in one place.
|
||||
#
|
||||
# Every door onto a semantic search faces the same question — which span of a
|
||||
# record do I show someone deciding whether to open it? A bare head cut shows
|
||||
# the document's OPENING, which is not the span that matched and not the span
|
||||
# the ranking was built on, and a cut per door gives each a different answer.
|
||||
#
|
||||
# Doors keep their own field names (the web UI renders `snippet`, the MCP
|
||||
# surface returns `excerpt`), because renaming a field a frontend consumes is
|
||||
# a separate change from fixing which text goes in it. What they share is this
|
||||
# function: the choice of span, and the obligation to say which span it is.
|
||||
|
||||
MATCHED_PASSAGE = "matched_passage"
|
||||
BODY_OPENING = "body_opening"
|
||||
|
||||
|
||||
def matched_excerpt(
|
||||
body: str, chunk: dict | None, budget: int
|
||||
) -> tuple[str, str, bool]:
|
||||
"""Pick the span to show, and say which span it is.
|
||||
|
||||
`chunk` is a `report["best_chunk"]` entry from one of the semantic
|
||||
searches — `{"index": int, "text": str}` — or None when the caller ran no
|
||||
semantic search, passed no report, or the record has no embedding row.
|
||||
|
||||
Returns `(text, kind, was_cut)` where `kind` is MATCHED_PASSAGE or
|
||||
BODY_OPENING. The kind is not decoration: a reader who cannot tell the
|
||||
passage that earned the hit from the first paragraph of the document
|
||||
cannot tell whether a thin-looking result is genuinely thin, and the whole
|
||||
point of the excerpt is to support exactly that judgement.
|
||||
|
||||
A short record comes back whole either way — fragmenting a 200-character
|
||||
note serves nobody, and its opening IS its content.
|
||||
"""
|
||||
passage = (chunk or {}).get("text") or ""
|
||||
if passage.strip():
|
||||
text, cut = elide(passage.strip(), budget)
|
||||
return text, MATCHED_PASSAGE, cut
|
||||
text, cut = elide(body or "", budget)
|
||||
return text, BODY_OPENING, cut
|
||||
|
||||
|
||||
def excerpt_fields(
|
||||
body: str, chunk: dict | None, budget: int, *, key: str = "excerpt"
|
||||
) -> dict:
|
||||
"""`matched_excerpt` as the block a result row carries.
|
||||
|
||||
`key` names the text field so a door can keep the name its consumers
|
||||
already read. The companion keys are derived from it, so a row never ends
|
||||
up with an excerpt under one name and its label under another.
|
||||
"""
|
||||
text, kind, cut = matched_excerpt(body, chunk, budget)
|
||||
out = {key: text, f"{key}_is": kind, "body_length": len(body or "")}
|
||||
if cut or (kind == MATCHED_PASSAGE and len(text) < len(body or "")):
|
||||
out["read_full"] = (
|
||||
"This is one span of a longer record. Open it by id for the whole "
|
||||
"thing rather than judging from what is here."
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Soft-delete / trash service — single source of truth for recoverable deletes.
|
||||
|
||||
A delete stamps `deleted_at` + a shared `deleted_batch_id` on the target row and
|
||||
its descendants (cascade). Restore clears the batch; purge hard-deletes it; the
|
||||
retention cron purges rows older than the configured window. Live reads exclude
|
||||
trashed rows via `alive()`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete as sql_delete, or_, select, update
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note import Note
|
||||
from grimoire.models.project import Project
|
||||
from grimoire.models.milestone import Milestone
|
||||
from grimoire.models.rulebook import Rulebook, RulebookTopic, Rule
|
||||
from grimoire.models.base import iso
|
||||
|
||||
# entity_type -> Model. Used to resolve which table a trash op targets.
|
||||
_MODEL_FOR = {
|
||||
"note": Note,
|
||||
"task": Note,
|
||||
"project": Project,
|
||||
"milestone": Milestone,
|
||||
"rulebook": Rulebook,
|
||||
"topic": RulebookTopic,
|
||||
"rule": Rule,
|
||||
}
|
||||
|
||||
|
||||
def _owner_clause(model, user_id: int):
|
||||
"""Boolean expr scoping `model` rows to the ones `user_id` owns.
|
||||
|
||||
EVERY trash query (exists-check, restore, purge, list, retention sweep)
|
||||
must carry this — a batch_id is a bearer token, so without an owner
|
||||
predicate a leaked/guessed id lets one tenant read, restore, or
|
||||
permanently destroy another's content. Topics and rules carry no
|
||||
user_id of their own; ownership is derived through the parent rulebook
|
||||
(or, for project-scoped rules, the owning project).
|
||||
"""
|
||||
if model is Rulebook:
|
||||
return Rulebook.owner_user_id == user_id
|
||||
if model is RulebookTopic:
|
||||
return RulebookTopic.rulebook_id.in_(
|
||||
select(Rulebook.id).where(Rulebook.owner_user_id == user_id)
|
||||
)
|
||||
if model is Rule:
|
||||
return or_(
|
||||
Rule.topic_id.in_(
|
||||
select(RulebookTopic.id)
|
||||
.join(Rulebook, RulebookTopic.rulebook_id == Rulebook.id)
|
||||
.where(Rulebook.owner_user_id == user_id)
|
||||
),
|
||||
Rule.project_id.in_(
|
||||
select(Project.id).where(Project.user_id == user_id)
|
||||
),
|
||||
)
|
||||
# Note, Project, Milestone all carry user_id directly.
|
||||
return model.user_id == user_id
|
||||
|
||||
|
||||
async def _set(session, model, where, batch, now) -> None:
|
||||
"""Stamp deleted_at + batch on live rows matching `where`."""
|
||||
await session.execute(
|
||||
update(model)
|
||||
.where(*where, model.deleted_at.is_(None))
|
||||
.values(deleted_at=now, deleted_batch_id=batch)
|
||||
)
|
||||
|
||||
|
||||
async def _exists_alive(session, user_id: int, etype: str, eid: int) -> bool:
|
||||
model = _MODEL_FOR[etype]
|
||||
where = [model.id == eid, model.deleted_at.is_(None), _owner_clause(model, user_id)]
|
||||
return (await session.execute(select(model.id).where(*where))).first() is not None
|
||||
|
||||
|
||||
async def _cascade(session, user_id: int, etype: str, eid: int, batch: str, now) -> None:
|
||||
if etype == "project":
|
||||
await _set(session, Note, [Note.user_id == user_id, Note.project_id == eid], batch, now)
|
||||
await _set(session, Milestone, [Milestone.user_id == user_id, Milestone.project_id == eid], batch, now)
|
||||
# Project-scoped rules cascade with the project they're attached to.
|
||||
await _set(session, Rule, [Rule.project_id == eid], batch, now)
|
||||
await _set(session, Project, [Project.user_id == user_id, Project.id == eid], batch, now)
|
||||
elif etype == "milestone":
|
||||
await _set(session, Note, [Note.user_id == user_id, Note.milestone_id == eid], batch, now)
|
||||
await _set(session, Milestone, [Milestone.user_id == user_id, Milestone.id == eid], batch, now)
|
||||
elif etype in ("note", "task"):
|
||||
# Stamp the entire sub-task subtree (not just direct children) so a
|
||||
# deeply nested task and all its descendants trash/restore as one batch.
|
||||
ids = [eid]
|
||||
frontier = [eid]
|
||||
while frontier:
|
||||
children = (await session.execute(
|
||||
select(Note.id).where(
|
||||
Note.user_id == user_id,
|
||||
Note.parent_id.in_(frontier),
|
||||
Note.deleted_at.is_(None),
|
||||
)
|
||||
)).scalars().all()
|
||||
frontier = [c for c in children if c not in ids]
|
||||
ids.extend(frontier)
|
||||
await _set(session, Note, [Note.user_id == user_id, Note.id.in_(ids)], batch, now)
|
||||
elif etype == "rulebook":
|
||||
topic_ids = (await session.execute(
|
||||
select(RulebookTopic.id)
|
||||
.join(Rulebook, RulebookTopic.rulebook_id == Rulebook.id)
|
||||
.where(Rulebook.id == eid, Rulebook.owner_user_id == user_id)
|
||||
)).scalars().all()
|
||||
if topic_ids:
|
||||
await _set(session, Rule, [Rule.topic_id.in_(topic_ids)], batch, now)
|
||||
await _set(session, RulebookTopic, [RulebookTopic.rulebook_id == eid], batch, now)
|
||||
await _set(session, Rulebook, [Rulebook.id == eid, Rulebook.owner_user_id == user_id], batch, now)
|
||||
elif etype == "topic":
|
||||
await _set(session, Rule, [Rule.topic_id == eid], batch, now)
|
||||
await _set(session, RulebookTopic, [RulebookTopic.id == eid], batch, now)
|
||||
elif etype == "rule":
|
||||
await _set(session, Rule, [Rule.id == eid], batch, now)
|
||||
else:
|
||||
raise ValueError(f"unknown entity_type: {etype!r}")
|
||||
|
||||
|
||||
async def delete(user_id: int, entity_type: str, entity_id: int) -> str | None:
|
||||
"""Soft-delete an entity + its descendants under one batch_id.
|
||||
|
||||
Returns the batch_id, or None if the entity wasn't found / not owned.
|
||||
"""
|
||||
batch = str(uuid.uuid4())
|
||||
now = datetime.now(timezone.utc)
|
||||
async with async_session() as session:
|
||||
if not await _exists_alive(session, user_id, entity_type, entity_id):
|
||||
return None
|
||||
await _cascade(session, user_id, entity_type, entity_id, batch, now)
|
||||
await session.commit()
|
||||
return batch
|
||||
|
||||
|
||||
# All soft-deletable models, and their trash-listing type label.
|
||||
_ALL = [Note, Project, Milestone, Rulebook, RulebookTopic, Rule]
|
||||
_TYPE = {
|
||||
Note: "note", Project: "project", Milestone: "milestone",
|
||||
Rulebook: "rulebook", RulebookTopic: "topic", Rule: "rule",
|
||||
}
|
||||
|
||||
|
||||
def alive(stmt, model):
|
||||
"""Append a 'not trashed' filter to a select. Use in every live read."""
|
||||
return stmt.where(model.deleted_at.is_(None))
|
||||
|
||||
|
||||
async def restore(user_id: int, batch_id: str) -> int:
|
||||
"""Clear deleted_at for every row in the batch. Returns rows restored."""
|
||||
n = 0
|
||||
async with async_session() as session:
|
||||
for model in _ALL:
|
||||
res = await session.execute(
|
||||
update(model)
|
||||
.where(model.deleted_batch_id == batch_id, _owner_clause(model, user_id))
|
||||
.values(deleted_at=None, deleted_batch_id=None)
|
||||
)
|
||||
n += res.rowcount or 0
|
||||
await session.commit()
|
||||
return n
|
||||
|
||||
|
||||
async def restore_entity(user_id: int, entity_type: str, entity_id: int) -> int | None:
|
||||
"""Restore ONE trashed entity by id, by reviving the batch that took it.
|
||||
|
||||
The inverse of `delete(user_id, entity_type, entity_id)`, which returns a
|
||||
batch id the caller usually doesn't keep. Callers that need to undo their own
|
||||
soft-delete later — snippet un-merge — know the entity id, not the
|
||||
batch, and looking it up here keeps them from reaching into the column set.
|
||||
|
||||
Restores the whole batch on purpose, not just the row: the batch is the
|
||||
entity plus its cascaded descendants, so reviving the parent alone would
|
||||
leave them orphaned in the trash. That is the same thing the trash UI does.
|
||||
|
||||
Returns rows restored, or None if the entity isn't trashed / not owned.
|
||||
"""
|
||||
model = next((m for m, label in _TYPE.items() if label == entity_type), None)
|
||||
if model is None:
|
||||
return None
|
||||
async with async_session() as session:
|
||||
batch = (await session.execute(
|
||||
select(model.deleted_batch_id).where(
|
||||
model.id == entity_id,
|
||||
_owner_clause(model, user_id),
|
||||
model.deleted_at.isnot(None),
|
||||
)
|
||||
)).scalars().first()
|
||||
if not batch:
|
||||
return None
|
||||
return await restore(user_id, batch)
|
||||
|
||||
|
||||
async def purge(user_id: int, batch_id: str) -> int:
|
||||
"""Hard-delete every row in the batch. Irreversible."""
|
||||
n = 0
|
||||
async with async_session() as session:
|
||||
for model in _ALL:
|
||||
res = await session.execute(
|
||||
sql_delete(model).where(
|
||||
model.deleted_batch_id == batch_id, _owner_clause(model, user_id)
|
||||
)
|
||||
)
|
||||
n += res.rowcount or 0
|
||||
await session.commit()
|
||||
return n
|
||||
|
||||
|
||||
async def list_trash(user_id: int) -> list[dict]:
|
||||
"""Trashed rows across all soft-deletable tables, grouped by batch_id."""
|
||||
batches: dict[str, dict] = {}
|
||||
async with async_session() as session:
|
||||
for model in _ALL:
|
||||
rows = (await session.execute(
|
||||
select(model).where(
|
||||
model.deleted_at.isnot(None), _owner_clause(model, user_id)
|
||||
)
|
||||
)).scalars().all()
|
||||
for r in rows:
|
||||
grp = batches.setdefault(
|
||||
r.deleted_batch_id,
|
||||
{"batch_id": r.deleted_batch_id,
|
||||
"deleted_at": iso(r.deleted_at),
|
||||
"items": []},
|
||||
)
|
||||
grp["items"].append({
|
||||
"type": _TYPE[model], "id": r.id,
|
||||
"title": getattr(r, "title", "") or "",
|
||||
})
|
||||
out = list(batches.values())
|
||||
for g in out:
|
||||
lead = g["items"][0]
|
||||
g["summary"] = lead["title"] or f"{lead['type']} {lead['id']}"
|
||||
g["count"] = len(g["items"])
|
||||
out.sort(key=lambda g: g["deleted_at"] or "", reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
async def purge_expired(user_id: int, retention_days: int) -> int:
|
||||
"""Cron entry: hard-delete THIS user's rows trashed more than retention_days ago.
|
||||
|
||||
Scoped to one owner so the scheduler can apply each user's own
|
||||
`trash_retention_days` window — a single global sweep would let one
|
||||
user's short window prematurely destroy another's data.
|
||||
retention_days <= 0 disables auto-purge (returns 0 without touching anything).
|
||||
"""
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
n = 0
|
||||
async with async_session() as session:
|
||||
for model in _ALL:
|
||||
res = await session.execute(
|
||||
sql_delete(model).where(
|
||||
model.deleted_at.isnot(None),
|
||||
model.deleted_at < cutoff,
|
||||
_owner_clause(model, user_id),
|
||||
)
|
||||
)
|
||||
n += res.rowcount or 0
|
||||
await session.commit()
|
||||
return n
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Daily APScheduler cron that purges expired trash.
|
||||
|
||||
A single job at 03:30 UTC (services/scheduler.py). Iterates every user and
|
||||
applies that user's own `trash_retention_days` setting; 0 disables auto-purge
|
||||
for that user.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from sqlalchemy import select
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.user import User
|
||||
from grimoire.services import trash as trash_svc
|
||||
from grimoire.services.scheduler import ScheduledJob
|
||||
from grimoire.services.settings import get_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _run_purge() -> None:
|
||||
async with async_session() as session:
|
||||
user_ids = (await session.execute(select(User.id))).scalars().all()
|
||||
|
||||
purged = 0
|
||||
for uid in user_ids:
|
||||
raw = await get_setting(uid, "trash_retention_days", "90")
|
||||
try:
|
||||
days = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
days = 90
|
||||
purged += await trash_svc.purge_expired(uid, days)
|
||||
if purged:
|
||||
logger.info("trash purge: removed %d expired row(s)", purged)
|
||||
else:
|
||||
logger.debug("trash purge: nothing expired")
|
||||
|
||||
|
||||
_JOB = ScheduledJob("trash_retention_purge", _run_purge, label="Trash retention")
|
||||
|
||||
|
||||
def start_trash_scheduler(loop: asyncio.AbstractEventLoop) -> None:
|
||||
_JOB.start(loop, CronTrigger(hour=3, minute=30, timezone="UTC"), describe="daily 03:30 UTC")
|
||||
|
||||
|
||||
def stop_trash_scheduler() -> None:
|
||||
_JOB.stop()
|
||||
@@ -0,0 +1,45 @@
|
||||
"""User-timezone helpers.
|
||||
|
||||
All datetimes in the DB are stored as UTC. The helpers here bridge between
|
||||
that UTC storage and the user's configured local timezone (IANA string in
|
||||
the ``user_timezone`` setting). Use these anywhere the model or UI talks
|
||||
in terms of "today", "tomorrow", or a bare calendar date — never
|
||||
``date.today()`` or ``datetime.now(timezone.utc)`` inside a per-user flow.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timedelta
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from grimoire.services.settings import get_setting
|
||||
|
||||
# Day-rollover boundary for day-anchored "today" logic. The "day" flips at
|
||||
# this local hour (not midnight) so the 00:00–04:00 local window still
|
||||
# shows yesterday's content until the user "starts" the new day.
|
||||
DAY_ROLLOVER_HOUR = 4
|
||||
|
||||
|
||||
async def get_user_tz(user_id: int) -> ZoneInfo:
|
||||
"""Return the user's IANA ``ZoneInfo``, falling back to UTC."""
|
||||
tz_str = await get_setting(user_id, "user_timezone") or "UTC"
|
||||
try:
|
||||
return ZoneInfo(tz_str)
|
||||
except (ZoneInfoNotFoundError, KeyError):
|
||||
return ZoneInfo("UTC")
|
||||
|
||||
|
||||
async def user_today(user_id: int) -> date:
|
||||
"""Return today's calendar date in the user's local timezone."""
|
||||
tz = await get_user_tz(user_id)
|
||||
return datetime.now(tz).date()
|
||||
|
||||
|
||||
async def user_day_date(user_id: int) -> date:
|
||||
"""Return the current "day" in the user's local timezone.
|
||||
|
||||
The day flips at ``DAY_ROLLOVER_HOUR`` (4am local) rather than midnight,
|
||||
so the 00:00–04:00 local window still returns *yesterday* — a day-anchored
|
||||
view stays on yesterday until the user crosses the rollover.
|
||||
"""
|
||||
tz = await get_user_tz(user_id)
|
||||
return (datetime.now(tz) - timedelta(hours=DAY_ROLLOVER_HOUR)).date()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""What a record's own check MEANS — shared by rules and notes.
|
||||
|
||||
Two record types carry `verify_with` / `expires_when` / `verified_at`: rules
|
||||
and notes. What they share is BEHAVIOUR — how a stamp is read, how old it is,
|
||||
what "never" means — not storage and not the query. The two sweeps stay
|
||||
separate siblings, because a rule scopes by rulebook ownership and a note
|
||||
scopes by the note ACL.
|
||||
These functions are the part that genuinely is common, factored so it exists
|
||||
once rather than twice.
|
||||
|
||||
Everything here is pure, sync and DUCK-TYPED: it reads `verify_with` and
|
||||
`verified_at` off whatever it is handed. That is deliberate. A shared base
|
||||
class or a protocol would tie two SQLAlchemy models together to share four
|
||||
lines of date arithmetic — the DRY costume, in the same note's words.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def last_verified_label(record) -> str | None:
|
||||
"""How long ago this record's check passed — None when it carries none.
|
||||
|
||||
Three states, and the distinction between the last two is the whole point:
|
||||
|
||||
- `None` — this record is a DECISION. There is nothing to go and check,
|
||||
and the question does not apply. Most records.
|
||||
- "never" — it asserts a fact and NOBODY HAS EVER CONFIRMED IT. The one
|
||||
worth acting on, and why the sweep sorts these first.
|
||||
- a date — somebody checked, then.
|
||||
|
||||
Callers attach this to a payload only when it is not None, so "no key" and
|
||||
"never verified" do not become two states every client has to tell apart.
|
||||
"""
|
||||
if not getattr(record, "verify_with", None):
|
||||
return None
|
||||
stamp = getattr(record, "verified_at", None)
|
||||
return stamp.date().isoformat() if stamp else "never"
|
||||
|
||||
|
||||
def days_since_verified(record) -> int | None:
|
||||
"""Whole days since the check last passed; None if it never has.
|
||||
|
||||
Computed rather than left to the reader, because "2026-06-14" and "74
|
||||
days" prompt different reactions and only one of them is the question
|
||||
being asked.
|
||||
|
||||
Naive stamps are read as UTC. Postgres hands these back tz-aware, but a
|
||||
restore, a fixture or a sqlite-backed test may not, and subtracting an
|
||||
aware datetime from a naive one raises — which would make the sweep fail
|
||||
on exactly the rows it exists to surface.
|
||||
"""
|
||||
stamp = getattr(record, "verified_at", None)
|
||||
if stamp is None:
|
||||
return None
|
||||
if stamp.tzinfo is None:
|
||||
stamp = stamp.replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - stamp).days
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Pin/unpin and auto-pin operations on NoteVersion.
|
||||
|
||||
The autosave-rolling system in `services/note_versions.py` continues to
|
||||
own the existing rolling-cap behavior; this module adds the manual-pin
|
||||
API and the daily auto-pin scan.
|
||||
|
||||
Tiers (pin_kind values):
|
||||
None → rolling autosave; capped at MAX_VERSIONS, FIFO.
|
||||
"auto" → system-declared via the stability scan; capped at MAX_AUTO_PINS, FIFO.
|
||||
"manual" → user-declared; unlimited, never pruned.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from datetime import timezone
|
||||
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from grimoire.models import async_session
|
||||
from grimoire.models.note_version import NoteVersion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_AUTO_PINS = 25
|
||||
AUTO_PIN_STABILITY_DAYS = 2
|
||||
PIN_LABEL_MAX_LEN = 500
|
||||
|
||||
|
||||
async def pin_version(
|
||||
user_id: int, note_id: int, version_id: int, *, label: str | None,
|
||||
) -> NoteVersion | None:
|
||||
"""Mark a version as manually pinned. Returns the updated row or None
|
||||
if not found (or wrong user/note scope).
|
||||
|
||||
Acceptable on already-pinned rows (manual or auto) — promotes to
|
||||
manual and updates the label. Labels are capped at PIN_LABEL_MAX_LEN
|
||||
chars; longer values raise ValueError.
|
||||
"""
|
||||
if label is not None and len(label) > PIN_LABEL_MAX_LEN:
|
||||
raise ValueError(
|
||||
f"pin_label too long ({len(label)} > {PIN_LABEL_MAX_LEN} chars)"
|
||||
)
|
||||
async with async_session() as session:
|
||||
version = (
|
||||
await session.execute(
|
||||
select(NoteVersion).where(
|
||||
NoteVersion.id == version_id,
|
||||
NoteVersion.note_id == note_id,
|
||||
NoteVersion.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalars().first()
|
||||
if version is None:
|
||||
return None
|
||||
version.pin_kind = "manual"
|
||||
version.pin_label = label
|
||||
await session.commit()
|
||||
await session.refresh(version)
|
||||
return version
|
||||
|
||||
|
||||
async def unpin_version(
|
||||
user_id: int, note_id: int, version_id: int,
|
||||
) -> NoteVersion | None:
|
||||
"""Clear pin_kind and pin_label, downgrading the row to rolling.
|
||||
|
||||
Does NOT delete the row. If the row is older than the rolling cap
|
||||
depth, the next create_version call will prune it via the rolling
|
||||
FIFO. Returns the updated row or None if not found.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
version = (
|
||||
await session.execute(
|
||||
select(NoteVersion).where(
|
||||
NoteVersion.id == version_id,
|
||||
NoteVersion.note_id == note_id,
|
||||
NoteVersion.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).scalars().first()
|
||||
if version is None:
|
||||
return None
|
||||
version.pin_kind = None
|
||||
version.pin_label = None
|
||||
await session.commit()
|
||||
await session.refresh(version)
|
||||
return version
|
||||
|
||||
|
||||
def _format_auto_pin_label(
|
||||
start: datetime.datetime, end: datetime.datetime | None,
|
||||
) -> str:
|
||||
"""Auto-generated label describing the stability window.
|
||||
|
||||
end=None → version is the latest with no successor; render as
|
||||
"stable since {start_iso}". Otherwise render as
|
||||
"stable {start_iso} → {end_iso}".
|
||||
"""
|
||||
s = start.date().isoformat()
|
||||
if end is None:
|
||||
return f"stable since {s}"
|
||||
return f"stable {s} → {end.date().isoformat()}"
|
||||
|
||||
|
||||
def _promote_stable_versions_for_note(versions_chrono: list) -> list:
|
||||
"""Mutate the input list: set pin_kind='auto' + auto-label on any
|
||||
unpinned version whose gap to its successor (or to now, for the
|
||||
latest) is >= AUTO_PIN_STABILITY_DAYS.
|
||||
|
||||
Returns the list of versions that were newly pinned (caller uses
|
||||
this for logging / counts).
|
||||
"""
|
||||
now = datetime.datetime.now(timezone.utc)
|
||||
newly_pinned: list = []
|
||||
for i, v in enumerate(versions_chrono):
|
||||
if v.pin_kind is not None:
|
||||
continue
|
||||
if i + 1 < len(versions_chrono):
|
||||
next_ts = versions_chrono[i + 1].created_at
|
||||
is_latest = False
|
||||
else:
|
||||
next_ts = now
|
||||
is_latest = True
|
||||
v_ts = v.created_at
|
||||
if v_ts.tzinfo is None:
|
||||
v_ts = v_ts.replace(tzinfo=timezone.utc)
|
||||
if next_ts.tzinfo is None:
|
||||
next_ts = next_ts.replace(tzinfo=timezone.utc)
|
||||
gap_days = (next_ts - v_ts).total_seconds() / 86400
|
||||
if gap_days >= AUTO_PIN_STABILITY_DAYS:
|
||||
v.pin_kind = "auto"
|
||||
v.pin_label = _format_auto_pin_label(
|
||||
v_ts, None if is_latest else next_ts,
|
||||
)
|
||||
newly_pinned.append(v)
|
||||
return newly_pinned
|
||||
|
||||
|
||||
async def _list_user_note_ids_with_versions(user_id: int) -> list[int]:
|
||||
"""Return note_ids belonging to user_id that have at least one row in
|
||||
note_versions. Notes with no version history are ignored."""
|
||||
async with async_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT DISTINCT note_id FROM note_versions
|
||||
WHERE user_id = :user_id
|
||||
"""
|
||||
).bindparams(user_id=user_id)
|
||||
)
|
||||
).all()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
async def _scan_one_note(user_id: int, note_id: int) -> int:
|
||||
"""Run the promote+prune flow for one note. Returns count of newly-
|
||||
pinned versions."""
|
||||
async with async_session() as session:
|
||||
versions = (
|
||||
await session.execute(
|
||||
select(NoteVersion)
|
||||
.where(
|
||||
NoteVersion.note_id == note_id,
|
||||
NoteVersion.user_id == user_id,
|
||||
)
|
||||
.order_by(NoteVersion.created_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
if not versions:
|
||||
return 0
|
||||
newly_pinned = _promote_stable_versions_for_note(list(versions))
|
||||
if newly_pinned:
|
||||
await session.commit()
|
||||
if newly_pinned:
|
||||
await prune_auto_pins(user_id=user_id, note_id=note_id)
|
||||
return len(newly_pinned)
|
||||
|
||||
|
||||
async def scan_user_for_auto_pins(user_id: int) -> int:
|
||||
"""Run the scan across every versioned note for one user. Returns
|
||||
total newly-pinned-this-pass."""
|
||||
total = 0
|
||||
note_ids = await _list_user_note_ids_with_versions(user_id)
|
||||
for note_id in note_ids:
|
||||
try:
|
||||
total += await _scan_one_note(user_id, note_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"auto-pin scan failed for user=%d note=%d", user_id, note_id,
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
async def scan_all_users_for_auto_pins() -> dict[int, int]:
|
||||
"""Top-level scan entrypoint. Iterates over all users and runs the
|
||||
per-user scan. Returns {user_id: newly_pinned_count}. Per-user errors
|
||||
are caught and logged so one user's failure doesn't stop the scan."""
|
||||
from grimoire.models import User
|
||||
|
||||
async with async_session() as session:
|
||||
users = (await session.execute(select(User.id))).scalars().all()
|
||||
|
||||
out: dict[int, int] = {}
|
||||
for uid in users:
|
||||
try:
|
||||
out[uid] = await scan_user_for_auto_pins(uid)
|
||||
except Exception:
|
||||
logger.exception("auto-pin scan failed for user=%d", uid)
|
||||
out[uid] = 0
|
||||
return out
|
||||
|
||||
|
||||
async def prune_auto_pins(user_id: int, note_id: int) -> None:
|
||||
"""FIFO-prune the auto-pinned bucket for one note past MAX_AUTO_PINS.
|
||||
|
||||
Manual pins and rolling rows are untouched. Called by the scan job
|
||||
after each note's auto-pin promotions finish.
|
||||
"""
|
||||
async with async_session() as session:
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM note_versions
|
||||
WHERE id IN (
|
||||
SELECT id FROM note_versions
|
||||
WHERE note_id = :note_id
|
||||
AND user_id = :user_id
|
||||
AND pin_kind = 'auto'
|
||||
ORDER BY created_at DESC
|
||||
OFFSET :max_auto_pins
|
||||
)
|
||||
"""
|
||||
).bindparams(
|
||||
note_id=note_id,
|
||||
user_id=user_id,
|
||||
max_auto_pins=MAX_AUTO_PINS,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Daily APScheduler cron for the auto-pin scan.
|
||||
|
||||
Single global job at 03:00 UTC. Runs scan_all_users_for_auto_pins so the
|
||||
system promotes stable note versions before they get aged out of the
|
||||
rolling cap. Off-hours by design — the scan is cheap but not time-
|
||||
critical and doesn't need to interrupt regular activity.
|
||||
|
||||
One ScheduledJob (services/scheduler.py), like the other *_scheduler modules.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from grimoire.services.scheduler import ScheduledJob
|
||||
from grimoire.services.version_pinning import scan_all_users_for_auto_pins
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _run_scan() -> None:
|
||||
results = await scan_all_users_for_auto_pins()
|
||||
total = sum(results.values())
|
||||
if total > 0:
|
||||
logger.info(
|
||||
"auto-pin scan: pinned %d version(s) across %d user(s)",
|
||||
total, len(results),
|
||||
)
|
||||
else:
|
||||
logger.debug("auto-pin scan: no new pins")
|
||||
|
||||
|
||||
_JOB = ScheduledJob("version_pinning_auto_scan", _run_scan, label="Version pinning")
|
||||
|
||||
|
||||
def start_version_pinning_scheduler(loop: asyncio.AbstractEventLoop) -> None:
|
||||
_JOB.start(loop, CronTrigger(hour=3, minute=0, timezone="UTC"), describe="daily 03:00 UTC")
|
||||
|
||||
|
||||
def stop_version_pinning_scheduler() -> None:
|
||||
_JOB.stop()
|
||||
Reference in New Issue
Block a user