feat(rules): a rule surfaces through the lessons confirmed as its instances (milestone 440 step 4, #4633)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 18s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / integration (push) Successful in 55s
CI & Build / Python tests (push) Failing after 1m16s
CI & Build / Build & push image (push) Skipped

On all three rule arms (prompt_rule, pre_tool_rule, write_path_rule), after
the direct match, a lesson matching the moment at the notes menu's own bar
brings the rule(s) it is CONFIRMED to be an instance of, rendered in rule
voice with "Reached through lesson #N “…”, a recorded instance of it."

- Confirmed links only (lesson_rules.confirmed_lessons /
  confirmed_rules_in_scope). A suggested link carrying its rule would
  manufacture the co-arrival #4637 counts and prove itself.
- Scope kept: a lesson surfaces everywhere, but the rule it brings must be
  global or this project's own — milestone 414's boundary, not reopened
  through a side door.
- Suppression is by RULE: anything the direct band named or the session
  ledger holds is skipped, whichever lesson reached it.
- Its own slot (VIA_LESSON_LIMIT = 1), not a rule slot — a stated default,
  since the plan's "decide by measurement" has nothing to measure until
  links are confirmed (#4632). Its own source, rule_via_lesson: registered,
  RANKED for pull-through, logged whenever it searches.
- Nothing searched at all while no lesson carries a confirmed link, which is
  every install until one is judged.
- build_prompt_rule_hint / build_tool_rule_hint are now thin wrappers over
  the direct arms (_prompt_rule_hint / _tool_rule_hint); the arm leaves its
  query in `_via_query` only when it ran, so a disabled arm or a blank prompt
  brings no rule in either, and the key never leaves the server.
- Via-lesson rules are not fed to the #4637 co-surfacing recorder: only
  direct matches are evidence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-01 15:03:41 -04:00
co-authored by Claude Opus 5.5
parent dbbab859ce
commit fd2ebf4c13
7 changed files with 448 additions and 3 deletions
+71 -1
View File
@@ -39,7 +39,7 @@ import logging
import re
from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, delete, exists, func, not_, select
from sqlalchemy import and_, delete, exists, func, not_, or_, select
from sqlalchemy.exc import IntegrityError
from scribe.models import async_session
@@ -650,3 +650,73 @@ async def _co_surfaced(user_id, lesson_ids, rule_ids, arm, situation, project_id
)).scalar_one_or_none() or ""
rule = owned[rid]
return _proposal_line(lid, lesson_title, rid, rule.title, rule.kind or "rule", ev)
# ── Reaching a rule through its lessons (#4633) ──────────────────────────────
async def confirmed_lessons(user_id: int) -> set[int]:
"""The lessons, readable by the caller, that carry a CONFIRMED link to a
rule the caller owns — the only lessons that can bring a rule along.
CONFIRMED ONLY, and the reason is the soft link's: a suggested link that
carried its rule would put the two side by side on every prompt that
found the lesson, which is exactly the co-arrival #4637 counts as
evidence — the suggestion would manufacture its own proof.
Asked first and cheaply, so an arm on an install with no confirmed links
— every install, until someone judges one — never pays for a search.
"""
from scribe.services.access import readable_notes_clause
from scribe.services.rulebooks import _owned_rules_clause
async with async_session() as session:
rows = (await session.execute(
select(LessonRuleLink.lesson_id)
.join(Rule, Rule.id == LessonRuleLink.rule_id)
.join(Note, Note.id == LessonRuleLink.lesson_id)
.where(LessonRuleLink.state == CONFIRMED)
.where(Rule.deleted_at.is_(None))
.where(_owned_rules_clause(user_id))
.where(Note.deleted_at.is_(None))
.where(readable_notes_clause(user_id))
.distinct()
)).scalars().all()
return {int(i) for i in rows}
async def confirmed_rules_in_scope(
user_id: int, lesson_ids, project_id: int | None,
) -> dict[int, list]:
"""{lesson_id: [Rule]} — each lesson's CONFIRMED rules that may surface here.
"May surface here" is the rule arms' own scope (milestone 414): a global
rule, or a rule of THIS project. A lesson is project-independent and
surfaces everywhere, so without this a lesson linked to project A's rule
would carry that rule into project B's sessions — the leak milestone 414
closed for the direct arms, reopened through a side door.
"""
from scribe.services.rulebooks import _owned_rules_clause
ids = [int(i) for i in lesson_ids or []]
if not ids:
return {}
home = (
or_(Rule.topic_id.isnot(None), Rule.project_id == project_id)
if project_id else Rule.topic_id.isnot(None)
)
async with async_session() as session:
rows = (await session.execute(
select(LessonRuleLink.lesson_id, Rule)
.join(Rule, Rule.id == LessonRuleLink.rule_id)
.where(LessonRuleLink.lesson_id.in_(ids))
.where(LessonRuleLink.state == CONFIRMED)
.where(Rule.deleted_at.is_(None))
.where(_owned_rules_clause(user_id))
.where(home)
.order_by(Rule.id)
)).all()
out: dict[int, list] = {}
for lesson_id, rule in rows:
out.setdefault(int(lesson_id), []).append(rule)
return out
+180
View File
@@ -1390,6 +1390,126 @@ async def _reserve_slot_for_preference(
return hits + slot, slot_id
# ── A rule reached through its lessons (milestone 440, #4633) ──────────────
#
# A rule's own document is written in the rule's words, which are general by
# design; the situations that keep proving it are often closer to what a
# session is actually doing. A lesson JUDGED to be an instance of a rule (a
# CONFIRMED link) carries that situation, so a lesson matching the moment
# brings its rule along — in rule voice, naming the lesson that reached it.
#
# ITS OWN SLOT, NOT A RULE SLOT, as a stated default. The plan asked for this
# to be settled by measurement, and there is nothing to measure until links
# are confirmed (#4632). Until then the asymmetry decides it: a via-lesson
# line that took a rule slot could push out a rule the ranker matched
# DIRECTLY, a stronger claim displaced by a weaker one, while an extra line
# costs one line. `rule_via_lesson` is logged as its own source so the
# question can be answered from data later (#4636).
VIA_LESSON_LIMIT = 1
# Lessons fetched before keeping only the linked ones. The search cannot be
# told "linked lessons only", so it overfetches and filters; on a corpus where
# most lessons are unlinked, a fetch of one would almost always be spent on a
# lesson that carries nothing.
_VIA_LESSON_OVERFETCH = 10
async def _rules_via_lessons(
user_id: int, query: str, *, project_id: int | None, skip: set[int],
held: set[int], where: str,
) -> tuple[list[str], list[int]]:
"""Rule lines reached through a matching lesson's CONFIRMED links.
The lesson bar is the notes menu's own threshold — the bar a lesson has to
clear to be shown at all — so a lesson too weak to surface cannot carry a
rule in. `skip` is every rule this response already names plus the
session ledger: suppression applies to the RULE, whichever lesson reached
it. Returns (lines, rule ids shown). Fails open, like every arm.
"""
try:
confirmed = await lesson_rules_svc.confirmed_lessons(user_id)
if not confirmed:
return [], []
bar = (await get_autoinject_config(user_id))["threshold"]
t0 = time.perf_counter()
_rep: dict = {}
found = await semantic_search_notes(
user_id, query, limit=_VIA_LESSON_OVERFETCH, threshold=bar,
project_id=project_id, note_type=(LESSON_NOTE_TYPE,),
include_global_kinds=True, scope="browse", report=_rep,
)
matched = [(s, n) for s, n in found if int(n.id) in confirmed]
by_lesson = await lesson_rules_svc.confirmed_rules_in_scope(
user_id, [int(n.id) for _s, n in matched], project_id,
)
candidates = [
(s, rule, n) for s, n in matched for rule in by_lesson.get(int(n.id), [])
]
chosen: list = []
taken: set[int] = set(skip)
for s, rule, n in candidates:
if rule.id in taken:
continue
taken.add(rule.id)
chosen.append((s, rule, n))
if len(chosen) >= VIA_LESSON_LIMIT:
break
# Logged whenever a search ran, results or not — the #3497 guard. The
# score is the LESSON's, because the lesson is what was matched; the
# best-available id is left off for the same reason, since the row's
# results are rules and an id beside them would read as a rule id.
record_retrieval(
user_id=user_id, source="rule_via_lesson", query=query,
threshold=bar, limit=VIA_LESSON_LIMIT, project_id=project_id,
is_task=None, results=[(s, rule) for s, rule, _n in chosen],
duration_ms=(time.perf_counter() - t0) * 1000.0,
best_available=_rep.get("best_available_score"),
searched=bool(_rep.get("searched", True)),
suppressed=len({r.id for _s, r, _n in candidates}) - len(chosen),
)
if not chosen:
return [], []
rule_ids = [rule.id for _s, rule, _n in chosen]
record_rule_surfaced(user_id=user_id, rule_ids=rule_ids, source="rule_via_lesson")
lines = [
_rule_hint_line(rule, where=where, seen=False, held=rule.id in held)
+ f" Reached through lesson #{n.id} "
+ f"\u201c{_menu_name(n.title, n.note_type, n.data, n.body)}\u201d, "
+ "a recorded instance of it."
for _s, rule, n in chosen
]
return lines, rule_ids
except Exception:
logger.debug("rule-via-lesson arm failed", exc_info=True)
return [], []
async def _add_rules_via_lessons(
user_id: int, out: dict, *, project_id: int, exclude_rule_ids, held_rule_ids,
where: str,
) -> dict:
"""Run the via-lesson step after a direct rule arm, on the query that arm
actually searched with.
The arm leaves `_via_query` in its payload only when it RAN — enabled, and
with something to search — so an arm the operator switched off, or a
blank prompt, brings no rule in through a side door either. The key is
popped here; it never leaves the server.
"""
query = out.pop("_via_query", "")
if not query:
return out
skip = (set(exclude_rule_ids or []) | set(out.get("rule_ids") or [])
| set(out.get("shown_rule_ids") or []))
lines, ids = await _rules_via_lessons(
user_id, query, project_id=project_id or None, skip=skip,
held=set(held_rule_ids or []), where=where,
)
if lines:
out["context"] = "\n".join(c for c in (out.get("context") or "", *lines) if c)
out["rule_ids"] = list(out.get("rule_ids") or []) + ids
return out
async def build_prompt_rule_hint(
user_id: int,
query: str,
@@ -1398,6 +1518,28 @@ async def build_prompt_rule_hint(
exclude_rule_ids: list[int] | None = None,
held_rule_ids: list[int] | None = None,
context: str = "",
) -> dict:
"""Rules and preferences that may apply to what the operator just asked —
matched directly (`_prompt_rule_hint`, where the design is written), then
reached through a linked lesson (`_rules_via_lessons`)."""
out = await _prompt_rule_hint(
user_id, query, project_id=project_id, exclude_rule_ids=exclude_rule_ids,
held_rule_ids=held_rule_ids, context=context,
)
return await _add_rules_via_lessons(
user_id, out, project_id=project_id, exclude_rule_ids=exclude_rule_ids,
held_rule_ids=held_rule_ids, where="to this request",
)
async def _prompt_rule_hint(
user_id: int,
query: str,
*,
project_id: int = 0,
exclude_rule_ids: list[int] | None = None,
held_rule_ids: list[int] | None = None,
context: str = "",
) -> dict:
"""Rules and preferences that may apply to what the operator just asked.
@@ -1436,6 +1578,9 @@ async def build_prompt_rule_hint(
# names it — and "yes, go ahead" names nothing a trigger can match, while
# the reply it answers ("commit this to dev and push") does.
q = _autoinject_query(q, context)
# The query this arm searches with, for the via-lesson step that follows
# it (#4633) — set only once the arm is going to run.
out["_via_query"] = q
try:
threshold = await floor_for(user_id, "prompt_rule")
@@ -2708,6 +2853,17 @@ async def build_write_path_hint(
except Exception:
logger.debug("write-path rule arm failed", exc_info=True)
# A rule reached through a linked lesson (#4633), after the direct band
# and never in place of it. Skips what the band already named and what
# the session's ledger holds — suppression is by rule.
via_lines, via_ids = await _rules_via_lessons(
user_id, code or path, project_id=project_id or None,
skip=set(exclude_rule_ids or []) | set(shown_rule_ids),
held=set(held_rule_ids or []), where="here",
)
lines.extend(via_lines)
rule_ids.extend(via_ids)
# A lesson and a rule on this one response (#4637): the soft-link
# recorder counts the pair, keyed on the FILE — every edit to one file is
# one situation — and asks about it once the evidence holds. Fails open.
@@ -2745,6 +2901,28 @@ async def build_tool_rule_hint(
project_id: int = 0,
exclude_rule_ids: list[int] | None = None,
held_rule_ids: list[int] | None = None,
) -> dict:
"""Standing rules that may apply to the action about to be taken — matched
directly (`_tool_rule_hint`, where the design is written), then reached
through a linked lesson (`_rules_via_lessons`, #4633)."""
out = await _tool_rule_hint(
user_id, tool_name, command, project_id=project_id,
exclude_rule_ids=exclude_rule_ids, held_rule_ids=held_rule_ids,
)
return await _add_rules_via_lessons(
user_id, out, project_id=project_id, exclude_rule_ids=exclude_rule_ids,
held_rule_ids=held_rule_ids, where=f"to this {tool_name} call",
)
async def _tool_rule_hint(
user_id: int,
tool_name: str,
command: str,
*,
project_id: int = 0,
exclude_rule_ids: list[int] | None = None,
held_rule_ids: list[int] | None = None,
) -> dict:
"""Standing rules that may apply to the ACTION about to be taken (#3476).
@@ -2795,6 +2973,8 @@ async def build_tool_rule_hint(
# embedding window, so it is bounded — the verb and its target sit at
# the front, which is the part a rule is about.
query = command[:_TOOL_QUERY_CHARS]
# For the via-lesson step (#4633) — set only once the arm is enabled.
out["_via_query"] = query
t0 = time.perf_counter()
_rep_ptr: dict = {}
@@ -137,6 +137,12 @@ POINTS: dict[str, Point] = dict([
"the one line reserved for a preference at the prompt boundary"),
_p("reuse_slot", UNBIDDEN, "the one line reserved for a reusable snippet"),
_p("lesson_slot", UNBIDDEN, "the one line reserved for a lesson"),
_p("rule_via_lesson", UNBIDDEN,
"a rule reached through a lesson confirmed as an instance of it",
expects_traffic=False,
quiet_because="searches only once some lesson has a confirmed link to "
"a rule; an install where none has been judged is "
"correctly silent here"),
# `fixed_query`: COMPLETION_QUERY is a module constant in
# services/reply_preferences.py, so this arm's top score is the same number
# on every call — measured at 0.791 across 45 consecutive calls, with p10,
+4
View File
@@ -110,6 +110,10 @@ RANKED_SOURCES = (
# The completion-report lookup on update_task (milestone 409 step 4). It
# runs its own query and shows only what cleared the bar — a ranker.
"report_preference",
# A rule reached through a lesson confirmed as an instance of it (#4633).
# Its own source so its pull-through reads apart from the rule's direct
# match — whether the lesson route earns its line is #4636's question.
"rule_via_lesson",
)
+7 -2
View File
@@ -157,7 +157,10 @@ def _no_lesson_rule_links(request):
otherwise ride along with every unit test that creates a lesson. The
co-surfacing recorder (#4637) is stubbed to "no proposal": every hook
builder test that renders a lesson beside a rule would otherwise reach
for the database to count the pair.
for the database to count the pair. And the via-lesson rule step (#4633)
is stubbed to "no lesson brought a rule": all three rule arms run it, and
its first act is a database read. tests/test_rule_via_lesson.py binds the
real function at import time, before this patch runs.
Skipped for integration tests, which exercise the real links against
Postgres (tests/test_integration_lesson_rule_links.py).
@@ -168,7 +171,9 @@ def _no_lesson_rule_links(request):
with patch("scribe.services.lesson_rules.attach_lesson_rules", AsyncMock()), \
patch("scribe.services.lesson_rules.attach_rule_lessons", AsyncMock()), \
patch("scribe.services.lesson_rules.rule_candidates", AsyncMock(return_value=[])), \
patch("scribe.services.lesson_rules.co_surfaced", AsyncMock(return_value="")):
patch("scribe.services.lesson_rules.co_surfaced", AsyncMock(return_value="")), \
patch("scribe.services.plugin_context._rules_via_lessons",
AsyncMock(return_value=([], []))):
yield
@@ -314,3 +314,36 @@ async def test_confirming_a_suggestion_keeps_what_it_rested_on(world):
out = await links_svc.judge_link(world["owner"], world["lesson"], world["r1"], "confirm", "same failure class")
assert out["state"] == "confirmed"
assert len(out["evidence"]["situations"]) == 3
# ── #4633: which links can carry a rule ─────────────────────────────────────
async def test_only_a_confirmed_link_can_carry_its_rule(world):
"""A suggested link never expands: it would manufacture the co-arrival
that #4637 counts, and prove itself. A rejected one obviously not."""
owner, lesson = world["owner"], world["lesson"]
await links_svc.co_surfaced(owner, [lesson], [world["r1"]], arm="p", situation="one ask here")
await links_svc.judge_link(owner, lesson, world["r2"], "reject", "not this")
assert await links_svc.confirmed_lessons(owner) == set()
assert await links_svc.confirmed_rules_in_scope(owner, [lesson], world["mine"]) == {}
await links_svc.judge_link(owner, lesson, world["r1"], "confirm", "same failure class")
assert lesson in await links_svc.confirmed_lessons(owner)
rules = await links_svc.confirmed_rules_in_scope(owner, [lesson], world["mine"])
assert [r.id for r in rules[lesson]] == [world["r1"]]
async def test_a_project_rule_stays_in_its_project_even_through_a_lesson(world):
"""Milestone 414's scope, kept on the side door: the lesson surfaces
everywhere, its project rule only in its project."""
owner, lesson = world["owner"], world["lesson"]
await links_svc.set_lesson_rules(owner, lesson, [world["r1"]])
assert await links_svc.confirmed_rules_in_scope(owner, [lesson], world["mine"])
assert await links_svc.confirmed_rules_in_scope(owner, [lesson], world["theirs"]) == {}
assert await links_svc.confirmed_rules_in_scope(owner, [lesson], None) == {}
async def test_a_stranger_reaches_no_rule_through_someone_elses_link(world):
await links_svc.set_lesson_rules(world["owner"], world["lesson"], [world["r1"]])
assert await links_svc.confirmed_lessons(world["stranger"]) == set()
+147
View File
@@ -0,0 +1,147 @@
"""A rule reached through its lessons (milestone 440, #4633).
The step runs after each of the three rule arms. These pin, with the database
and the embedder stubbed: that nothing is searched when no lesson carries a
confirmed link; that a matching linked lesson brings its rule in rule voice,
naming the lesson; that suppression is by RULE; that the slot holds one line;
and that the call is logged under its own source. That a SUGGESTED link never
expands — the soft link proving itself — is pinned against Postgres in
tests/test_integration_lesson_rule_links.py, where the state filter lives.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from scribe.services import plugin_context as pc
from scribe.services import rule_usage
from scribe.services.retrieval_registry import POINTS
# Bound before conftest's autouse stub replaces the module attribute.
_REAL = pc._rules_via_lessons
def _lesson(lid=41, title="An overrun run usually failed early"):
return SimpleNamespace(
id=lid, title=title, note_type="lesson", body="",
data={"what": title, "when_to_apply": "a CI run overran"},
)
def _rule(rid, title="Read the job log first", kind="rule"):
return SimpleNamespace(id=rid, title=title, kind=kind, when_to_apply="a run overran")
def _stubs(*, confirmed, found, by_lesson):
log, surfaced = MagicMock(), MagicMock()
stack = [
patch.object(pc.lesson_rules_svc, "confirmed_lessons", AsyncMock(return_value=confirmed)),
patch.object(pc.lesson_rules_svc, "confirmed_rules_in_scope", AsyncMock(return_value=by_lesson)),
patch.object(pc, "get_autoinject_config", AsyncMock(return_value={"threshold": 0.5})),
patch.object(pc, "semantic_search_notes", AsyncMock(return_value=found)),
patch.object(pc, "record_retrieval", log),
patch.object(pc, "record_rule_surfaced", surfaced),
]
return stack, log, surfaced
async def _run(stack, **kw):
for p in stack:
p.start()
try:
return await _REAL(1, "the CI run is still going", project_id=2,
skip=kw.get("skip", set()), held=set(), where="to this request")
finally:
for p in stack:
p.stop()
@pytest.mark.asyncio
async def test_no_confirmed_link_means_no_search_and_no_row():
search = AsyncMock()
stack, log, _ = _stubs(confirmed=set(), found=[], by_lesson={})
stack[3] = patch.object(pc, "semantic_search_notes", search)
assert await _run(stack) == ([], [])
search.assert_not_awaited()
log.assert_not_called()
@pytest.mark.asyncio
async def test_a_matching_linked_lesson_brings_its_rule_in_rule_voice():
stack, log, surfaced = _stubs(
confirmed={41}, found=[(0.71, _lesson())], by_lesson={41: [_rule(7)]},
)
lines, ids = await _run(stack)
assert ids == [7]
assert lines[0].startswith("Standing rule")
assert "Reached through lesson #41" in lines[0]
assert log.call_args.kwargs["source"] == "rule_via_lesson"
assert surfaced.call_args.kwargs == {"user_id": 1, "rule_ids": [7], "source": "rule_via_lesson"}
@pytest.mark.asyncio
async def test_a_lesson_without_a_confirmed_link_carries_nothing():
"""The search found a lesson, but it is not in the confirmed set — a
suggested or unlinked lesson brings no rule."""
stack, log, surfaced = _stubs(
confirmed={99}, found=[(0.9, _lesson(41))], by_lesson={},
)
assert await _run(stack) == ([], [])
surfaced.assert_not_called()
assert log.call_args.kwargs["results"] == []
@pytest.mark.asyncio
async def test_suppression_is_by_rule_whichever_lesson_reached_it():
stack, log, surfaced = _stubs(
confirmed={41}, found=[(0.71, _lesson())], by_lesson={41: [_rule(7)]},
)
assert await _run(stack, skip={7}) == ([], [])
assert log.call_args.kwargs["suppressed"] == 1
surfaced.assert_not_called()
@pytest.mark.asyncio
async def test_the_slot_holds_one_line():
stack, _log, _ = _stubs(
confirmed={41, 42},
found=[(0.8, _lesson(41)), (0.7, _lesson(42, "Another"))],
by_lesson={41: [_rule(7), _rule(8)], 42: [_rule(9)]},
)
lines, ids = await _run(stack)
assert ids == [7] and len(lines) == pc.VIA_LESSON_LIMIT == 1
@pytest.mark.asyncio
async def test_a_failure_brings_no_rule_and_raises_nothing():
stack, _log, _ = _stubs(confirmed={41}, found=[], by_lesson={})
stack[0] = patch.object(pc.lesson_rules_svc, "confirmed_lessons",
AsyncMock(side_effect=RuntimeError("db down")))
assert await _run(stack) == ([], [])
@pytest.mark.asyncio
async def test_the_step_runs_only_when_the_arm_ran_and_never_leaks_its_key():
step = AsyncMock(return_value=(["Standing rule … Reached through lesson #41"], [7]))
with patch.object(pc, "_rules_via_lessons", step):
idle = await pc._add_rules_via_lessons(
1, {"context": "", "rule_ids": []}, project_id=2,
exclude_rule_ids=[3], held_rule_ids=[], where="here",
)
ran = await pc._add_rules_via_lessons(
1, {"context": "direct line", "rule_ids": [5], "_via_query": "q"},
project_id=2, exclude_rule_ids=[3], held_rule_ids=[], where="here",
)
assert idle == {"context": "", "rule_ids": []}
assert step.await_count == 1
assert step.await_args.kwargs["skip"] == {3, 5}
assert "_via_query" not in ran
assert ran["rule_ids"] == [5, 7]
assert ran["context"].startswith("direct line\n")
def test_the_source_is_ranked_and_registered():
assert "rule_via_lesson" in rule_usage.RANKED_SOURCES
assert "rule_via_lesson" in POINTS