feat(rulings): system_usage_events is read back — per-System counts and a telemetry block (#4769)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Successful in 50s
CI & Build / TypeScript typecheck (push) Successful in 52s
CI & Build / Python tests (push) Successful in 1m44s
CI & Build / Build & push image (push) Successful in 36s

#4769 "Rulings are counted where someone will read them": milestone 444
step 4 wrote system_usage_events and nothing read it.

- retrieval_telemetry gains a `system_usage` block: surfacings and opens by
  source, distinct counts, and `by_system` naming the areas most shown.
  There is deliberately no pull-through ratio, because rulings travel in full
  in the line and opens are the exception.
- usage_for_systems (one GROUP BY) adds `usage` to the REST Systems list and
  detail, and to MCP get_system. MCP list_systems is unchanged.
- The Systems UI shows a "rulings shown N×" chip.
- rulings_pre_tool, rulings_write_path and mcp_get_system are now declared
  registry points; the registry guard covers their recorders.
- The Systems store merges a PATCH reply instead of replacing the row.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 08:49:20 -04:00
co-authored by Claude Opus 5.5
parent 1b973ebd13
commit 0a1bb68808
12 changed files with 457 additions and 4 deletions
+120 -1
View File
@@ -34,6 +34,10 @@ from scribe.models.rule_usage import DEPARTED as RULE_DEPARTED
from scribe.models.rule_usage import PULLED as RULE_PULLED
from scribe.models.rule_usage import SURFACED as RULE_SURFACED
from scribe.models.rule_usage import RuleUsageEvent
from scribe.models.system import System
from scribe.models.system_usage import PULLED as SYSTEM_PULLED
from scribe.models.system_usage import SURFACED as SYSTEM_SURFACED
from scribe.models.system_usage import SystemUsageEvent
from scribe.services.rule_usage import is_ambient
from scribe.models.retrieval_log import RetrievalLog
from scribe.services.retrieval_registry import (
@@ -817,7 +821,7 @@ def _compute_warnings(sources: dict, usage: dict, rule_usage: dict,
def _silent_surfaces(sources: dict, usage: dict, rule_usage: dict,
active: bool) -> list[dict]:
active: bool, system_usage: dict | None = None) -> list[dict]:
"""Registered points that emitted nothing at all in the window.
THE HALF THE ROWS CANNOT SEE. Every check above reads rows, so an arm that
@@ -834,12 +838,124 @@ def _silent_surfaces(sources: dict, usage: dict, rule_usage: dict,
return []
seen = set(sources) | set((usage.get("by_source") or {}))
seen |= set((rule_usage.get("by_source") or {}))
seen |= set(((system_usage or {}).get("by_source") or {}))
return [
{"source": s, "kind": POINTS[s].kind, "what": POINTS[s].what}
for s in sources_expected_to_emit() if s not in seen
]
# How many Systems `by_system` lists. A readout of which AREAS' rulings are
# reaching sessions, not an export: an install with more ruled areas than this
# reads the rest from the per-System counts on the Systems list.
SYSTEM_USAGE_TOP = 20
async def _system_usage(user_id: int | None, since) -> dict:
"""The rulings arm's own block (#4769): were an area's rulings shown, and
was the area then opened.
FROM `system_usage_events`, IN ITS OWN SESSION AND ITS OWN GUARD, for the
reason the rule block is guarded apart: a failure here must not take down
the readouts beside it, and must say so rather than print zeros (#2663) —
the keys are kept and `system_usage_failed` is added.
NO PULL-THROUGH, and the omission is the design. The note and rule ratios
ask "was the hint any use" of a line that carries a TITLE, where opening
the record is how it gets read. A rulings line carries the rulings
themselves; the agent reads them without opening anything, and
`get_system` is reached for only when a long list was cut ("+N more").
A ratio here would read near zero on an arm that is working, and invite
the dead-weight verdict the per-record badges give — the wrong conclusion,
built from a true number. The counts are printed side by side instead.
`by_system` names the areas, because "rulings were shown 40 times" says
nothing a reader can act on, and "Billing, 38 of them" does.
Ordered by surfacings, then by name, so a stable window reads stably.
"""
block: dict = {
"surfaced": 0,
"pulled": 0,
"distinct_systems_surfaced": 0,
"distinct_systems_pulled": 0,
"by_source": {},
"by_system": [],
}
try:
async with async_session() as session:
where = (
SystemUsageEvent.created_at >= since,
SystemUsageEvent.user_id == user_id,
)
per_source = (
await session.execute(
select(
SystemUsageEvent.event,
SystemUsageEvent.source,
func.count().label("n"),
)
.where(*where)
.group_by(SystemUsageEvent.event, SystemUsageEvent.source)
)
).all()
per_system = (
await session.execute(
select(
SystemUsageEvent.system_id,
System.name,
SystemUsageEvent.event,
func.count().label("n"),
func.max(SystemUsageEvent.created_at).label("last_at"),
)
# Outer: telemetry outlives the row it describes, and a
# deleted System's surfacings still happened.
.outerjoin(System, System.id == SystemUsageEvent.system_id)
.where(*where)
.group_by(
SystemUsageEvent.system_id, System.name,
SystemUsageEvent.event,
)
)
).all()
complete = await _complete_from(session, SystemUsageEvent, user_id)
except Exception:
logger.warning("system usage read failed", exc_info=True)
block["system_usage_failed"] = True
return block
for event, source, n in per_source:
n = int(n)
key = "surfaced" if event == SYSTEM_SURFACED else (
"pulled" if event == SYSTEM_PULLED else None)
if key is None:
continue
block[key] += n
block["by_source"][source] = block["by_source"].get(source, 0) + n
systems: dict[int, dict] = {}
for system_id, name, event, n, last_at in per_system:
row = systems.setdefault(int(system_id), {
"system_id": int(system_id),
"name": name,
"surfaced": 0,
"pulled": 0,
"last_surfaced_at": None,
})
if event == SYSTEM_SURFACED:
row["surfaced"] += int(n)
row["last_surfaced_at"] = iso(last_at)
elif event == SYSTEM_PULLED:
row["pulled"] += int(n)
block["distinct_systems_surfaced"] = sum(1 for r in systems.values() if r["surfaced"])
block["distinct_systems_pulled"] = sum(1 for r in systems.values() if r["pulled"])
block["by_system"] = sorted(
(r for r in systems.values() if r["surfaced"]),
key=lambda r: (-r["surfaced"], r["name"] or ""),
)[:SYSTEM_USAGE_TOP]
block.update(_coverage(complete.get("*"), since))
return block
async def retrieval_summary(
user_id: int | None, *, days: int = 30, near_miss_samples: int = 0,
) -> dict:
@@ -878,6 +994,7 @@ async def retrieval_summary(
"sources": {},
"usage": {},
"rule_usage": {},
"system_usage": {},
"read_failed": False,
}
@@ -1435,6 +1552,7 @@ async def retrieval_summary(
)
rule_usage.update(_coverage((rule_complete or {}).get("*"), since))
out["rule_usage"] = rule_usage
out["system_usage"] = await _system_usage(user_id, since)
# ── What is wrong (#3431) ────────────────────────────────────────────
#
@@ -1514,6 +1632,7 @@ async def retrieval_summary(
)
out["silent_surfaces"] = _silent_surfaces(
out["sources"], usage, rule_usage, active,
system_usage=out["system_usage"],
)
return out