feat(moments): the human door onto mounts, mappings and per-moment telemetry (milestone 458 step 6, #4924)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / Python tests (push) Failing after 1m34s
CI & Build / Build & push image (push) Skipped
CI & Build / integration (push) Successful in 2m7s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / Python tests (push) Failing after 1m34s
CI & Build / Build & push image (push) Skipped
CI & Build / integration (push) Successful in 2m7s
Everything an agent can do with moments, a person can now see and change in the app. - Rule editor: a moment picker beside the trigger. Catalog moments are ticked; a named procedure's `skill.<name>` is typed and checked as the server checks it. `moments` is always sent, so unticking the last moment unmounts the rule. - Settings, Moments section (General tab): for each moment, what it means, the actions that reach it on this install (shipped ones can be switched off, the install's own removed), how many rules are mounted on it, and deliveries and agent opens over the window. Below that: named procedures with mounts, switched-off defaults with Restore, and a form to add an action. - retrieval_telemetry.moment_usage: per moment, `delivered`, `rules`, `opened` (agent pulls after the first delivery there; an upper bound, as by_source is) and `last_delivered_at`. No ratio, because a mount is a person's statement, not a ranker's guess. Guarded on its own, and also reported in retrieval_summary as `moment_usage`. - rulebooks.mount_counts; mounted_moments now derives from it. - GET /api/retrieval/moments carries `mounted` and `usage` (?days=). DELETE /moments/mappings also reads the mapping from query parameters, since the browser's DELETE sends no body. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,8 @@ from typing import Any
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy import case, exists, func, select
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from scribe.models import async_session
|
||||
from scribe.models.base import iso
|
||||
@@ -985,6 +986,94 @@ async def _system_usage(user_id: int | None, since) -> dict:
|
||||
return block
|
||||
|
||||
|
||||
|
||||
async def moment_usage(user_id: int | None, since) -> dict:
|
||||
"""Deliveries and opens per moment (milestone 458 step 6).
|
||||
|
||||
A mounted rule's delivery is a `moment_rule` surfacing whose `detail` is
|
||||
the moment it arrived at — the plugin's catch-all hook, the MCP tools'
|
||||
attached lines and the reply hold all record it so. Per moment:
|
||||
|
||||
- `delivered`: how many times a mounted rule arrived there;
|
||||
- `rules`: how many distinct rules did;
|
||||
- `opened`: how many of those an AGENT then opened (`mcp_*`), at or after
|
||||
their first delivery there in the window;
|
||||
- `last_delivered_at`.
|
||||
|
||||
`opened` is an UPPER BOUND, for by_source's reason: a pull records the
|
||||
door, not the line that prompted it, so a rule delivered at two moments
|
||||
and opened once counts as opened at both. A moment reading near zero is
|
||||
not being flattered by it.
|
||||
|
||||
NO RATIO. A mount is a person's statement that the rule belongs at that
|
||||
moment, not a ranker's guess, so "delivered often, opened rarely" is a
|
||||
question about the rule's wording or the agent, not a bar to tune. The
|
||||
counts sit side by side.
|
||||
|
||||
Its own session and guard (#2663): a failure keeps the shape and adds
|
||||
`moment_usage_failed`, never zeros that read as "nothing happened".
|
||||
"""
|
||||
from scribe.services.retrieval_pipeline import MOMENT_RULE_SOURCE
|
||||
|
||||
block: dict = {"by_moment": {}}
|
||||
try:
|
||||
async with async_session() as session:
|
||||
per_rule = (
|
||||
select(
|
||||
RuleUsageEvent.detail.label("moment"),
|
||||
RuleUsageEvent.rule_id,
|
||||
func.count().label("n"),
|
||||
func.min(RuleUsageEvent.created_at).label("first_at"),
|
||||
func.max(RuleUsageEvent.created_at).label("last_at"),
|
||||
)
|
||||
.where(
|
||||
RuleUsageEvent.created_at >= since,
|
||||
RuleUsageEvent.user_id == user_id,
|
||||
RuleUsageEvent.event == RULE_SURFACED,
|
||||
RuleUsageEvent.source == MOMENT_RULE_SOURCE,
|
||||
RuleUsageEvent.detail.is_not(None),
|
||||
)
|
||||
.group_by(RuleUsageEvent.detail, RuleUsageEvent.rule_id)
|
||||
.subquery()
|
||||
)
|
||||
pull = aliased(RuleUsageEvent)
|
||||
opened = exists().where(
|
||||
pull.rule_id == per_rule.c.rule_id,
|
||||
pull.user_id == user_id,
|
||||
pull.event == RULE_PULLED,
|
||||
# autoescape: `_` is a LIKE wildcard (see the note block).
|
||||
pull.source.startswith("mcp_", autoescape=True),
|
||||
pull.created_at >= per_rule.c.first_at,
|
||||
)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
per_rule.c.moment,
|
||||
func.sum(per_rule.c.n).label("delivered"),
|
||||
func.count().label("rules"),
|
||||
func.count(case((opened, 1))).label("opened"),
|
||||
func.max(per_rule.c.last_at).label("last_at"),
|
||||
).group_by(per_rule.c.moment)
|
||||
)
|
||||
).all()
|
||||
complete = await _complete_from(session, RuleUsageEvent, user_id)
|
||||
except Exception:
|
||||
logger.warning("moment usage read failed", exc_info=True)
|
||||
block["moment_usage_failed"] = True
|
||||
return block
|
||||
|
||||
block["by_moment"] = {
|
||||
moment: {
|
||||
"delivered": int(delivered or 0),
|
||||
"rules": int(rules or 0),
|
||||
"opened": int(n_opened or 0),
|
||||
"last_delivered_at": iso(last_at),
|
||||
}
|
||||
for moment, delivered, rules, n_opened, last_at in rows
|
||||
}
|
||||
block.update(_coverage(complete.get("*"), since))
|
||||
return block
|
||||
|
||||
async def retrieval_summary(
|
||||
user_id: int | None, *, days: int = 30, near_miss_samples: int = 0,
|
||||
) -> dict:
|
||||
@@ -1583,6 +1672,7 @@ async def retrieval_summary(
|
||||
rule_usage.update(_coverage((rule_complete or {}).get("*"), since))
|
||||
out["rule_usage"] = rule_usage
|
||||
out["system_usage"] = await _system_usage(user_id, since)
|
||||
out["moment_usage"] = await moment_usage(user_id, since)
|
||||
|
||||
# ── Judged lines (#4772) ─────────────────────────────────────────────
|
||||
#
|
||||
|
||||
@@ -1208,6 +1208,16 @@ async def mounted_moments(user_id: int) -> set[str]:
|
||||
tool whose moments carry nothing. A superset is the safe direction — the
|
||||
delivery read still scopes by project.
|
||||
"""
|
||||
return set(await mount_counts(user_id))
|
||||
|
||||
|
||||
async def mount_counts(user_id: int) -> dict[str, int]:
|
||||
"""How many of this caller's live rules are mounted on each moment.
|
||||
|
||||
The same read as `mounted_moments`, counted — what the Settings view
|
||||
shows beside each moment, so a moment carrying nothing reads as such.
|
||||
Moments with no mount are absent rather than zero.
|
||||
"""
|
||||
from scribe.models.rulebook import rule_moments as rule_moments_t
|
||||
from scribe.services.rule_scope import joined_to_homes, rule_home
|
||||
|
||||
@@ -1215,13 +1225,14 @@ async def mounted_moments(user_id: int) -> set[str]:
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
joined_to_homes(
|
||||
select(rule_moments_t.c.moment).distinct()
|
||||
select(rule_moments_t.c.moment, func.count(func.distinct(Rule.id)))
|
||||
.select_from(rule_moments_t)
|
||||
.join(Rule, Rule.id == rule_moments_t.c.rule_id)
|
||||
)
|
||||
.where(Rule.deleted_at.is_(None), home)
|
||||
)).scalars().all()
|
||||
return set(rows)
|
||||
.group_by(rule_moments_t.c.moment)
|
||||
)).all()
|
||||
return {moment: int(n) for moment, n in rows}
|
||||
|
||||
|
||||
async def list_rule_systems(rule_ids: list[int]) -> dict[int, list[dict]]:
|
||||
|
||||
Reference in New Issue
Block a user