feat(moments): the human door onto mounts, mappings and per-moment telemetry (milestone 458 step 6, #4924)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / Python tests (push) Failing after 1m34s
CI & Build / Build & push image (push) Skipped
CI & Build / integration (push) Successful in 2m7s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / Python tests (push) Failing after 1m34s
CI & Build / Build & push image (push) Skipped
CI & Build / integration (push) Successful in 2m7s
Everything an agent can do with moments, a person can now see and change in the app. - Rule editor: a moment picker beside the trigger. Catalog moments are ticked; a named procedure's `skill.<name>` is typed and checked as the server checks it. `moments` is always sent, so unticking the last moment unmounts the rule. - Settings, Moments section (General tab): for each moment, what it means, the actions that reach it on this install (shipped ones can be switched off, the install's own removed), how many rules are mounted on it, and deliveries and agent opens over the window. Below that: named procedures with mounts, switched-off defaults with Restore, and a form to add an action. - retrieval_telemetry.moment_usage: per moment, `delivered`, `rules`, `opened` (agent pulls after the first delivery there; an upper bound, as by_source is) and `last_delivered_at`. No ratio, because a mount is a person's statement, not a ranker's guess. Guarded on its own, and also reported in retrieval_summary as `moment_usage`. - rulebooks.mount_counts; mounted_moments now derives from it. - GET /api/retrieval/moments carries `mounted` and `usage` (?days=). DELETE /moments/mappings also reads the mapping from query parameters, since the browser's DELETE sends no body. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -13,12 +13,15 @@ So every change to a retrieval dial goes through `set_dial`, from the UI as
|
||||
much as from the MCP tool, and the only difference is the `actor` recorded.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from scribe.auth import get_current_user_id, login_required
|
||||
from scribe.services import moment_actions as moment_actions_svc
|
||||
from scribe.services import moments as moments_svc
|
||||
from scribe.services import rulebooks as rulebooks_svc
|
||||
from scribe.services.retrieval_telemetry import moment_usage
|
||||
from scribe.services.retrieval_tuning import set_dial, current_settings, tuning_history
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -115,16 +118,39 @@ async def moments_route():
|
||||
actions that reach each one on this install.
|
||||
|
||||
The same payload `list_moments` returns, from the same services, so the
|
||||
session and the Settings view cannot name different moments.
|
||||
session and the Settings view cannot name different moments — plus what
|
||||
the view shows beside each one (step 6): `mounted`, how many rules each
|
||||
carries, and `usage`, the deliveries and opens per moment over `days`
|
||||
(the block `retrieval_telemetry` reports as `moment_usage`).
|
||||
"""
|
||||
uid = get_current_user_id()
|
||||
try:
|
||||
days = max(1, min(int(request.args.get("days", 30)), 365))
|
||||
except ValueError:
|
||||
return jsonify({"error": "days must be a whole number"}), 400
|
||||
out = moments_svc.catalog()
|
||||
out.update(await moment_actions_svc.actions_by_moment(get_current_user_id()))
|
||||
out.update(await moment_actions_svc.actions_by_moment(uid))
|
||||
# The catalog is the page; the counts beside it are not worth losing it
|
||||
# over. A failed count is flagged rather than shown as "nothing mounted".
|
||||
try:
|
||||
out["mounted"] = await rulebooks_svc.mount_counts(uid)
|
||||
except Exception:
|
||||
logger.warning("mount counts unreadable", exc_info=True)
|
||||
out["mounted"], out["mounted_failed"] = {}, True
|
||||
out["usage"] = await moment_usage(uid, datetime.now(timezone.utc) - timedelta(days=days))
|
||||
out["usage"]["days"] = days
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
async def _mapping_change(change):
|
||||
"""map/unmap from the browser: the MCP tools' service, recorded as human."""
|
||||
data = await request.get_json()
|
||||
"""map/unmap from the browser: the MCP tools' service, recorded as human.
|
||||
|
||||
The mapping comes as a JSON body, or — for DELETE, whose body many
|
||||
clients will not send — as query parameters of the same names.
|
||||
"""
|
||||
data = await request.get_json(silent=True)
|
||||
if data is None and request.args:
|
||||
data = request.args.to_dict()
|
||||
if not isinstance(data, dict):
|
||||
return jsonify({"error": "Expected a JSON object"}), 400
|
||||
if not data.get("tool") or not data.get("moment"):
|
||||
|
||||
@@ -22,7 +22,8 @@ from typing import Any
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy import case, exists, func, select
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from scribe.models import async_session
|
||||
from scribe.models.base import iso
|
||||
@@ -985,6 +986,94 @@ async def _system_usage(user_id: int | None, since) -> dict:
|
||||
return block
|
||||
|
||||
|
||||
|
||||
async def moment_usage(user_id: int | None, since) -> dict:
|
||||
"""Deliveries and opens per moment (milestone 458 step 6).
|
||||
|
||||
A mounted rule's delivery is a `moment_rule` surfacing whose `detail` is
|
||||
the moment it arrived at — the plugin's catch-all hook, the MCP tools'
|
||||
attached lines and the reply hold all record it so. Per moment:
|
||||
|
||||
- `delivered`: how many times a mounted rule arrived there;
|
||||
- `rules`: how many distinct rules did;
|
||||
- `opened`: how many of those an AGENT then opened (`mcp_*`), at or after
|
||||
their first delivery there in the window;
|
||||
- `last_delivered_at`.
|
||||
|
||||
`opened` is an UPPER BOUND, for by_source's reason: a pull records the
|
||||
door, not the line that prompted it, so a rule delivered at two moments
|
||||
and opened once counts as opened at both. A moment reading near zero is
|
||||
not being flattered by it.
|
||||
|
||||
NO RATIO. A mount is a person's statement that the rule belongs at that
|
||||
moment, not a ranker's guess, so "delivered often, opened rarely" is a
|
||||
question about the rule's wording or the agent, not a bar to tune. The
|
||||
counts sit side by side.
|
||||
|
||||
Its own session and guard (#2663): a failure keeps the shape and adds
|
||||
`moment_usage_failed`, never zeros that read as "nothing happened".
|
||||
"""
|
||||
from scribe.services.retrieval_pipeline import MOMENT_RULE_SOURCE
|
||||
|
||||
block: dict = {"by_moment": {}}
|
||||
try:
|
||||
async with async_session() as session:
|
||||
per_rule = (
|
||||
select(
|
||||
RuleUsageEvent.detail.label("moment"),
|
||||
RuleUsageEvent.rule_id,
|
||||
func.count().label("n"),
|
||||
func.min(RuleUsageEvent.created_at).label("first_at"),
|
||||
func.max(RuleUsageEvent.created_at).label("last_at"),
|
||||
)
|
||||
.where(
|
||||
RuleUsageEvent.created_at >= since,
|
||||
RuleUsageEvent.user_id == user_id,
|
||||
RuleUsageEvent.event == RULE_SURFACED,
|
||||
RuleUsageEvent.source == MOMENT_RULE_SOURCE,
|
||||
RuleUsageEvent.detail.is_not(None),
|
||||
)
|
||||
.group_by(RuleUsageEvent.detail, RuleUsageEvent.rule_id)
|
||||
.subquery()
|
||||
)
|
||||
pull = aliased(RuleUsageEvent)
|
||||
opened = exists().where(
|
||||
pull.rule_id == per_rule.c.rule_id,
|
||||
pull.user_id == user_id,
|
||||
pull.event == RULE_PULLED,
|
||||
# autoescape: `_` is a LIKE wildcard (see the note block).
|
||||
pull.source.startswith("mcp_", autoescape=True),
|
||||
pull.created_at >= per_rule.c.first_at,
|
||||
)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
per_rule.c.moment,
|
||||
func.sum(per_rule.c.n).label("delivered"),
|
||||
func.count().label("rules"),
|
||||
func.count(case((opened, 1))).label("opened"),
|
||||
func.max(per_rule.c.last_at).label("last_at"),
|
||||
).group_by(per_rule.c.moment)
|
||||
)
|
||||
).all()
|
||||
complete = await _complete_from(session, RuleUsageEvent, user_id)
|
||||
except Exception:
|
||||
logger.warning("moment usage read failed", exc_info=True)
|
||||
block["moment_usage_failed"] = True
|
||||
return block
|
||||
|
||||
block["by_moment"] = {
|
||||
moment: {
|
||||
"delivered": int(delivered or 0),
|
||||
"rules": int(rules or 0),
|
||||
"opened": int(n_opened or 0),
|
||||
"last_delivered_at": iso(last_at),
|
||||
}
|
||||
for moment, delivered, rules, n_opened, last_at in rows
|
||||
}
|
||||
block.update(_coverage(complete.get("*"), since))
|
||||
return block
|
||||
|
||||
async def retrieval_summary(
|
||||
user_id: int | None, *, days: int = 30, near_miss_samples: int = 0,
|
||||
) -> dict:
|
||||
@@ -1583,6 +1672,7 @@ async def retrieval_summary(
|
||||
rule_usage.update(_coverage((rule_complete or {}).get("*"), since))
|
||||
out["rule_usage"] = rule_usage
|
||||
out["system_usage"] = await _system_usage(user_id, since)
|
||||
out["moment_usage"] = await moment_usage(user_id, since)
|
||||
|
||||
# ── Judged lines (#4772) ─────────────────────────────────────────────
|
||||
#
|
||||
|
||||
@@ -1208,6 +1208,16 @@ async def mounted_moments(user_id: int) -> set[str]:
|
||||
tool whose moments carry nothing. A superset is the safe direction — the
|
||||
delivery read still scopes by project.
|
||||
"""
|
||||
return set(await mount_counts(user_id))
|
||||
|
||||
|
||||
async def mount_counts(user_id: int) -> dict[str, int]:
|
||||
"""How many of this caller's live rules are mounted on each moment.
|
||||
|
||||
The same read as `mounted_moments`, counted — what the Settings view
|
||||
shows beside each moment, so a moment carrying nothing reads as such.
|
||||
Moments with no mount are absent rather than zero.
|
||||
"""
|
||||
from scribe.models.rulebook import rule_moments as rule_moments_t
|
||||
from scribe.services.rule_scope import joined_to_homes, rule_home
|
||||
|
||||
@@ -1215,13 +1225,14 @@ async def mounted_moments(user_id: int) -> set[str]:
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(
|
||||
joined_to_homes(
|
||||
select(rule_moments_t.c.moment).distinct()
|
||||
select(rule_moments_t.c.moment, func.count(func.distinct(Rule.id)))
|
||||
.select_from(rule_moments_t)
|
||||
.join(Rule, Rule.id == rule_moments_t.c.rule_id)
|
||||
)
|
||||
.where(Rule.deleted_at.is_(None), home)
|
||||
)).scalars().all()
|
||||
return set(rows)
|
||||
.group_by(rule_moments_t.c.moment)
|
||||
)).all()
|
||||
return {moment: int(n) for moment, n in rows}
|
||||
|
||||
|
||||
async def list_rule_systems(rule_ids: list[int]) -> dict[int, list[dict]]:
|
||||
|
||||
Reference in New Issue
Block a user