feat(rulings): system_usage_events is read back — per-System counts and a telemetry block (#4769)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Successful in 50s
CI & Build / TypeScript typecheck (push) Successful in 52s
CI & Build / Python tests (push) Successful in 1m44s
CI & Build / Build & push image (push) Successful in 36s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Successful in 50s
CI & Build / TypeScript typecheck (push) Successful in 52s
CI & Build / Python tests (push) Successful in 1m44s
CI & Build / Build & push image (push) Successful in 36s
#4769 "Rulings are counted where someone will read them": milestone 444 step 4 wrote system_usage_events and nothing read it. - retrieval_telemetry gains a `system_usage` block: surfacings and opens by source, distinct counts, and `by_system` naming the areas most shown. There is deliberately no pull-through ratio, because rulings travel in full in the line and opens are the exception. - usage_for_systems (one GROUP BY) adds `usage` to the REST Systems list and detail, and to MCP get_system. MCP list_systems is unchanged. - The Systems UI shows a "rulings shown N×" chip. - rulings_pre_tool, rulings_write_path and mcp_get_system are now declared registry points; the registry guard covers their recorders. - The Systems store merges a PATCH reply instead of replacing the row. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -34,6 +34,10 @@ from scribe.models.rule_usage import DEPARTED as RULE_DEPARTED
|
||||
from scribe.models.rule_usage import PULLED as RULE_PULLED
|
||||
from scribe.models.rule_usage import SURFACED as RULE_SURFACED
|
||||
from scribe.models.rule_usage import RuleUsageEvent
|
||||
from scribe.models.system import System
|
||||
from scribe.models.system_usage import PULLED as SYSTEM_PULLED
|
||||
from scribe.models.system_usage import SURFACED as SYSTEM_SURFACED
|
||||
from scribe.models.system_usage import SystemUsageEvent
|
||||
from scribe.services.rule_usage import is_ambient
|
||||
from scribe.models.retrieval_log import RetrievalLog
|
||||
from scribe.services.retrieval_registry import (
|
||||
@@ -817,7 +821,7 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
|
||||
|
||||
|
||||
def _silent_surfaces(sources: dict, usage: dict, rule_usage: dict,
|
||||
active: bool) -> list[dict]:
|
||||
active: bool, system_usage: dict | None = None) -> list[dict]:
|
||||
"""Registered points that emitted nothing at all in the window.
|
||||
|
||||
THE HALF THE ROWS CANNOT SEE. Every check above reads rows, so an arm that
|
||||
@@ -834,12 +838,124 @@ def _silent_surfaces(sources: dict, usage: dict, rule_usage: dict,
|
||||
return []
|
||||
seen = set(sources) | set((usage.get("by_source") or {}))
|
||||
seen |= set((rule_usage.get("by_source") or {}))
|
||||
seen |= set(((system_usage or {}).get("by_source") or {}))
|
||||
return [
|
||||
{"source": s, "kind": POINTS[s].kind, "what": POINTS[s].what}
|
||||
for s in sources_expected_to_emit() if s not in seen
|
||||
]
|
||||
|
||||
|
||||
# How many Systems `by_system` lists. A readout of which AREAS' rulings are
|
||||
# reaching sessions, not an export: an install with more ruled areas than this
|
||||
# reads the rest from the per-System counts on the Systems list.
|
||||
SYSTEM_USAGE_TOP = 20
|
||||
|
||||
|
||||
async def _system_usage(user_id: int | None, since) -> dict:
|
||||
"""The rulings arm's own block (#4769): were an area's rulings shown, and
|
||||
was the area then opened.
|
||||
|
||||
FROM `system_usage_events`, IN ITS OWN SESSION AND ITS OWN GUARD, for the
|
||||
reason the rule block is guarded apart: a failure here must not take down
|
||||
the readouts beside it, and must say so rather than print zeros (#2663) —
|
||||
the keys are kept and `system_usage_failed` is added.
|
||||
|
||||
NO PULL-THROUGH, and the omission is the design. The note and rule ratios
|
||||
ask "was the hint any use" of a line that carries a TITLE, where opening
|
||||
the record is how it gets read. A rulings line carries the rulings
|
||||
themselves; the agent reads them without opening anything, and
|
||||
`get_system` is reached for only when a long list was cut ("+N more").
|
||||
A ratio here would read near zero on an arm that is working, and invite
|
||||
the dead-weight verdict the per-record badges give — the wrong conclusion,
|
||||
built from a true number. The counts are printed side by side instead.
|
||||
|
||||
`by_system` names the areas, because "rulings were shown 40 times" says
|
||||
nothing a reader can act on, and "Billing, 38 of them" does.
|
||||
Ordered by surfacings, then by name, so a stable window reads stably.
|
||||
"""
|
||||
block: dict = {
|
||||
"surfaced": 0,
|
||||
"pulled": 0,
|
||||
"distinct_systems_surfaced": 0,
|
||||
"distinct_systems_pulled": 0,
|
||||
"by_source": {},
|
||||
"by_system": [],
|
||||
}
|
||||
try:
|
||||
async with async_session() as session:
|
||||
where = (
|
||||
SystemUsageEvent.created_at >= since,
|
||||
SystemUsageEvent.user_id == user_id,
|
||||
)
|
||||
per_source = (
|
||||
await session.execute(
|
||||
select(
|
||||
SystemUsageEvent.event,
|
||||
SystemUsageEvent.source,
|
||||
func.count().label("n"),
|
||||
)
|
||||
.where(*where)
|
||||
.group_by(SystemUsageEvent.event, SystemUsageEvent.source)
|
||||
)
|
||||
).all()
|
||||
per_system = (
|
||||
await session.execute(
|
||||
select(
|
||||
SystemUsageEvent.system_id,
|
||||
System.name,
|
||||
SystemUsageEvent.event,
|
||||
func.count().label("n"),
|
||||
func.max(SystemUsageEvent.created_at).label("last_at"),
|
||||
)
|
||||
# Outer: telemetry outlives the row it describes, and a
|
||||
# deleted System's surfacings still happened.
|
||||
.outerjoin(System, System.id == SystemUsageEvent.system_id)
|
||||
.where(*where)
|
||||
.group_by(
|
||||
SystemUsageEvent.system_id, System.name,
|
||||
SystemUsageEvent.event,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
complete = await _complete_from(session, SystemUsageEvent, user_id)
|
||||
except Exception:
|
||||
logger.warning("system usage read failed", exc_info=True)
|
||||
block["system_usage_failed"] = True
|
||||
return block
|
||||
|
||||
for event, source, n in per_source:
|
||||
n = int(n)
|
||||
key = "surfaced" if event == SYSTEM_SURFACED else (
|
||||
"pulled" if event == SYSTEM_PULLED else None)
|
||||
if key is None:
|
||||
continue
|
||||
block[key] += n
|
||||
block["by_source"][source] = block["by_source"].get(source, 0) + n
|
||||
|
||||
systems: dict[int, dict] = {}
|
||||
for system_id, name, event, n, last_at in per_system:
|
||||
row = systems.setdefault(int(system_id), {
|
||||
"system_id": int(system_id),
|
||||
"name": name,
|
||||
"surfaced": 0,
|
||||
"pulled": 0,
|
||||
"last_surfaced_at": None,
|
||||
})
|
||||
if event == SYSTEM_SURFACED:
|
||||
row["surfaced"] += int(n)
|
||||
row["last_surfaced_at"] = iso(last_at)
|
||||
elif event == SYSTEM_PULLED:
|
||||
row["pulled"] += int(n)
|
||||
block["distinct_systems_surfaced"] = sum(1 for r in systems.values() if r["surfaced"])
|
||||
block["distinct_systems_pulled"] = sum(1 for r in systems.values() if r["pulled"])
|
||||
block["by_system"] = sorted(
|
||||
(r for r in systems.values() if r["surfaced"]),
|
||||
key=lambda r: (-r["surfaced"], r["name"] or ""),
|
||||
)[:SYSTEM_USAGE_TOP]
|
||||
block.update(_coverage(complete.get("*"), since))
|
||||
return block
|
||||
|
||||
|
||||
async def retrieval_summary(
|
||||
user_id: int | None, *, days: int = 30, near_miss_samples: int = 0,
|
||||
) -> dict:
|
||||
@@ -878,6 +994,7 @@ async def retrieval_summary(
|
||||
"sources": {},
|
||||
"usage": {},
|
||||
"rule_usage": {},
|
||||
"system_usage": {},
|
||||
"read_failed": False,
|
||||
}
|
||||
|
||||
@@ -1435,6 +1552,7 @@ async def retrieval_summary(
|
||||
)
|
||||
rule_usage.update(_coverage((rule_complete or {}).get("*"), since))
|
||||
out["rule_usage"] = rule_usage
|
||||
out["system_usage"] = await _system_usage(user_id, since)
|
||||
|
||||
# ── What is wrong (#3431) ────────────────────────────────────────────
|
||||
#
|
||||
@@ -1514,6 +1632,7 @@ async def retrieval_summary(
|
||||
)
|
||||
out["silent_surfaces"] = _silent_surfaces(
|
||||
out["sources"], usage, rule_usage, active,
|
||||
system_usage=out["system_usage"],
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user