bvandeusen/FabledScribe
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

wip(394): steps 6+7 — backend path and instruction surfaces

Browse Source
This commit is contained in:
bvandeusen
2026-09-11 15:15:33 -04:00
parent 690ca0306e
commit c149ef31a3
28 changed files with 260 additions and 738 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scribe/services/backup.py
+9 -7
View File
@@ -513,7 +513,7 @@ def _rule_version_rows(rows) -> list[dict]:
"id": rv.id, "rule_id": rv.rule_id, "user_id": rv.user_id,
"title": rv.title, "statement": rv.statement, "why": rv.why,
"how_to_apply": rv.how_to_apply, "when_to_apply": rv.when_to_apply,
"tier": rv.tier, "kind": rv.kind, "verify_with": rv.verify_with,
"kind": rv.kind, "verify_with": rv.verify_with,
"expires_when": rv.expires_when,
"created_at": rv.created_at.isoformat(),
}
@@ -529,7 +529,7 @@ def _rulebook_rows(rows) -> list[dict]:
return [
{
"id": rb.id, "owner_user_id": rb.owner_user_id, "title": rb.title,
"description": rb.description, "always_on": rb.always_on,
"description": rb.description,
"created_at": rb.created_at.isoformat(),
"updated_at": rb.updated_at.isoformat(),
}
@@ -575,7 +575,7 @@ def _rule_rows(rows) -> list[dict]:
"id": r.id, "topic_id": r.topic_id, "project_id": r.project_id,
"title": r.title, "statement": r.statement, "why": r.why,
"how_to_apply": r.how_to_apply, "order_index": r.order_index,
"when_to_apply": r.when_to_apply, "tier": r.tier, "kind": r.kind,
"when_to_apply": r.when_to_apply, "kind": r.kind,
"verify_with": r.verify_with, "expires_when": r.expires_when,
"verified_at": r.verified_at.isoformat() if r.verified_at else None,
"arose_from_id": r.arose_from_id,
@@ -1238,7 +1238,6 @@ async def _restore_v2(data: dict) -> dict:
owner_user_id=mapped_uid,
title=rb_data.get("title", ""),
description=rb_data.get("description", ""),
always_on=rb_data.get("always_on", False),
created_at=_dt(rb_data.get("created_at")),
updated_at=_dt(rb_data.get("updated_at")),
)
@@ -1279,10 +1278,14 @@ async def _restore_v2(data: dict) -> dict:
why=r_data.get("why") or None,
how_to_apply=r_data.get("how_to_apply") or None,
when_to_apply=r_data.get("when_to_apply") or None,
# A file written before migration 0088 has no tier. always_on
# A file written before milestone 394 carries `tier` and
# `always_on`; neither is read. Dropping a field the schema
# no longer has is the tolerant direction — an archive
# records what WAS, and refusing it because it remembers a
# deleted column would make every pre-394 backup
# unrestorable. Previously: always_on
# is the pre-0088 behaviour, so an old backup restores rules
# that bind exactly as they did when it was taken.
tier=r_data.get("tier") or "always_on",
# Same shape, same reason: a file written before 0098 has no
# kind, and every rule in it was a rule. Defaulting the other
# way would restore an old backup with things that had always
@@ -1427,7 +1430,6 @@ async def _restore_v2(data: dict) -> dict:
why=rv.get("why"),
how_to_apply=rv.get("how_to_apply"),
when_to_apply=rv.get("when_to_apply"),
tier=rv.get("tier"),
# NOT defaulted, unlike the rule above. A version records what
# was; absent means nobody wrote it down, and inventing "rule"
# here would put an artifact where a measurement belongs.
src/scribe/services/embeddings.py
+7 -13
View File
@@ -817,7 +817,6 @@ async def semantic_search_rules(
query: str,
limit: int = 5,
threshold: float = _SIMILARITY_THRESHOLD,
tier: str | None = None,
kind: str | None = None,
report: dict | None = None,
) -> list[tuple[float, "Rule"]]:
@@ -844,17 +843,13 @@ async def semantic_search_rules(
is the surfacing question, and it has its own machinery
(get_applicable_rules) rather than a second, subtly different copy here.
`tier` narrows to one tier, and NONE is the ordinary case. The write-path
and pre-tool hints deliberately pass nothing: an always-on rule is already
in the session, but being in a list from turn zero is not the same as being
in front of the reader when the action it governs is taken, and filtering
on tier made a whole class of rules permanently ineligible for the one
mechanism that surfaces a rule AT the moment. Relevance is the threshold's
job; see the block above RULEHINT_LIMIT in services/plugin_context.py for
the argument and for what the resulting scores are being read against.
Pass a tier when a caller genuinely wants one class — a listing, an audit,
a UI that renders the tiers apart. Not to approximate relevance.
THERE IS NO TIER TO NARROW BY ANY MORE (milestone 394). This carried a
`tier` parameter, and the arms deliberately passed nothing: filtering on it
made a whole class of rules permanently ineligible for the one mechanism
that surfaces a rule AT the moment it applies. The tier is now gone
entirely, so every rule is eligible for every arm and relevance is the
threshold's job alone — see the block above RULEHINT_LIMIT in
services/plugin_context.py for what those scores are read against.
`kind` narrows to `rule` or `preference`, and NONE is likewise the ordinary
case: a caller asking "what governs this" wants both, because the reader
@@ -905,7 +900,6 @@ async def semantic_search_rules(
Rulebook.owner_user_id == user_id,
Project.user_id == user_id,
),
*( [Rule.tier == tier] if tier else [] ),
*( [Rule.kind == kind] if kind else [] ),
)
# Overfetch so collapsing chunks to their best row still fills
src/scribe/services/inception.py
+32 -45
View File
@@ -7,7 +7,6 @@ A project's inheritance is a decision, not a default. The record lives on
"decided_at": "<iso>", "decided_by": <user id> | null,
"via": "mcp" | "ui" | "legacy",
"choices": {
"exclude_always_on_rulebooks": [rulebook ids],
"subscribe_rulebooks": [rulebook ids],
"design_system_id": <id> | null,
"seed_systems": bool
@@ -18,9 +17,14 @@ NULL = undecided → enter_project asks. ``legacy`` is the migration's stamp on
projects that existed before the step did (inherit-all / no design system /
no seed), so the ask fires only for projects created after this shipped.
``exclude_always_on_rulebooks`` was a fourth choice until milestone 394. It
let a project decline to inherit an always-on rulebook, and with no always-on
tier there is nothing to decline — a rulebook now reaches a project by
subscription, which is opt-IN, so declining is expressed by not subscribing.
The shape and its validator are pure; ``decide`` composes the existing
services — always-on exclusions, subscriptions, set_project_design_system,
the standard Systems seed — checks every target BEFORE touching anything,
services — subscriptions, set_project_design_system, the standard Systems
seed — checks every target BEFORE touching anything,
applies the effects (each idempotent), and writes the record LAST, so a
half-applied decision is re-runnable rather than recorded as done.
``current_defaults`` is what the enter_project ask shows: what binds today
@@ -37,7 +41,7 @@ from scribe.models.project import Project
from scribe.models.rulebook import Rulebook
INCEPTION_VIAS = ("mcp", "ui", "legacy")
CHOICE_KEYS = ("exclude_always_on_rulebooks", "subscribe_rulebooks", "design_system_id", "seed_systems")
CHOICE_KEYS = ("subscribe_rulebooks", "design_system_id", "seed_systems")
def _is_id_list(value) -> bool:
@@ -60,15 +64,9 @@ def validate_inception(choices) -> str | None:
unknown = sorted(set(choices) - set(CHOICE_KEYS))
if unknown:
return f"unknown inception choice(s): {', '.join(unknown)} (one of: {', '.join(CHOICE_KEYS)})"
excl = choices.get("exclude_always_on_rulebooks") or []
subs = choices.get("subscribe_rulebooks") or []
if not _is_id_list(excl):
return "exclude_always_on_rulebooks must be a list of rulebook ids"
if not _is_id_list(subs):
return "subscribe_rulebooks must be a list of rulebook ids"
both = sorted(set(excl) & set(subs))
if both:
return f"rulebook(s) {both} cannot be both excluded and subscribed"
ds = choices.get("design_system_id")
if ds is not None and (isinstance(ds, bool) or not isinstance(ds, int) or ds <= 0):
return "design_system_id must be a positive id or null"
@@ -79,11 +77,10 @@ def validate_inception(choices) -> str | None:
def normalize_choices(choices: dict | None) -> dict:
"""The four keys, always present, in canonical form — what gets stored
"""The three keys, always present, in canonical form — what gets stored
and what the UI/agent reads back. Call after validate_inception."""
choices = choices or {}
return {
"exclude_always_on_rulebooks": sorted(set(choices.get("exclude_always_on_rulebooks") or [])),
"subscribe_rulebooks": sorted(set(choices.get("subscribe_rulebooks") or [])),
"design_system_id": choices.get("design_system_id"),
"seed_systems": bool(choices.get("seed_systems", False)),
@@ -98,8 +95,7 @@ def is_decided(project) -> bool:
async def current_defaults(user_id: int, project_id: int) -> dict:
"""What the project inherits if nobody decides — the ask's payload.
{always_on_rulebooks: [{id,title}], other_rulebooks: [{id,title}],
excluded_always_on: [...], subscribed_rulebooks: [...],
{rulebooks: [{id,title}], subscribed_rulebooks: [...],
design_system_id, design_systems: [{id,title}], systems: <count>}.
Instance-agnostic: an install with no rulebooks / design systems shows
empty lists, and the ask says so rather than inventing a default.
@@ -115,7 +111,7 @@ async def current_defaults(user_id: int, project_id: int) -> dict:
async with async_session() as session:
rows = (
await session.execute(
select(Rulebook.id, Rulebook.title, Rulebook.always_on)
select(Rulebook.id, Rulebook.title)
.where(Rulebook.owner_user_id == user_id, Rulebook.deleted_at.is_(None))
.order_by(Rulebook.title)
)
@@ -124,9 +120,11 @@ async def current_defaults(user_id: int, project_id: int) -> dict:
designs = await design_systems_svc.list_design_systems(user_id)
systems = await systems_svc.list_systems(user_id, project_id, include_archived=True)
return {
"always_on_rulebooks": [{"id": i, "title": t} for i, t, on in rows if on],
"other_rulebooks": [{"id": i, "title": t} for i, t, on in rows if not on],
"excluded_always_on": applicable.get("excluded_always_on", []),
# ONE list since milestone 394. This was split into always-on and
# "other" because the first bound the project whether it asked or not;
# with the tier gone every rulebook is opt-in, so the split named a
# difference that no longer exists.
"rulebooks": [{"id": i, "title": t} for i, t in rows],
"subscribed_rulebooks": applicable.get("subscribed_rulebooks", []),
"design_system_id": project.design_system_id,
"design_systems": [{"id": d.id, "title": d.title} for d in designs],
@@ -139,28 +137,22 @@ async def _check_targets(user_id: int, choices: dict) -> None:
effect lands — a decision applies whole or errors whole."""
from scribe.services import access
wanted = set(choices["exclude_always_on_rulebooks"]) | set(choices["subscribe_rulebooks"])
wanted = set(choices["subscribe_rulebooks"])
if wanted:
async with async_session() as session:
rows = (
await session.execute(
select(Rulebook.id, Rulebook.always_on).where(
select(Rulebook.id).where(
Rulebook.id.in_(wanted),
Rulebook.owner_user_id == user_id,
Rulebook.deleted_at.is_(None),
)
)
).all()
found = {rid: on for rid, on in rows}
missing = sorted(wanted - set(found))
found = {rid for (rid,) in rows}
missing = sorted(wanted - found)
if missing:
raise ValueError(f"rulebook(s) {missing} not found (or not yours)")
not_always = sorted(r for r in choices["exclude_always_on_rulebooks"] if not found[r])
if not_always:
raise ValueError(
f"rulebook(s) {not_always} are not always-on — only always-on rulebooks "
"can be excluded; a subscribed rulebook is simply not subscribed"
)
ds = choices["design_system_id"]
if ds is not None and not await access.can_read_design_system(user_id, ds):
raise ValueError(f"design system {ds} not found (or not readable)")
@@ -176,13 +168,12 @@ async def decide(
"""Record a project's inception decision and apply it (milestone 297).
Owner-only. Validates the choices (pure) and every target (owned /
readable) first; then, each idempotent: exclude the named always-on
rulebooks, subscribe the named rulebooks, point the project at the design
system (None = explicitly none), seed the standard Systems if asked and
the project has none; then write ``projects.inception`` LAST. Re-deciding
is additive for exclusions/subscriptions (nothing is silently dropped —
include/unsubscribe are explicit calls), replaces the design system, and
re-seeds nothing a project already has.
readable) first; then, each idempotent: subscribe the named rulebooks,
point the project at the design system (None = explicitly none), seed the
standard Systems if asked and the project has none; then write
``projects.inception`` LAST. Re-deciding is additive for subscriptions
(nothing is silently dropped — unsubscribe is an explicit call), replaces
the design system, and re-seeds nothing a project already has.
Returns {"inception": <record>, "effects": {excluded, subscribed,
design_system_id, systems_seeded}}.
@@ -203,8 +194,6 @@ async def decide(
raise ValueError(f"project {project_id} not found (or not yours)")
await _check_targets(user_id, choices)
for rb in choices["exclude_always_on_rulebooks"]:
await rulebooks_svc.exclude_always_on_rulebook_for_project(project_id, rb, user_id)
for rb in choices["subscribe_rulebooks"]:
await rulebooks_svc.subscribe_project(project_id, rb, user_id)
if not await design_systems_svc.set_project_design_system(
@@ -230,7 +219,6 @@ async def decide(
return {
"inception": record,
"effects": {
"excluded": choices["exclude_always_on_rulebooks"],
"subscribed": choices["subscribe_rulebooks"],
"design_system_id": choices["design_system_id"],
"systems_seeded": [sy.name for sy in seeded],
@@ -247,25 +235,24 @@ async def inception_ask(user_id: int, project_id: int) -> dict:
defaults = await current_defaults(user_id, project_id)
except Exception:
return {}
always = ", ".join(f"{r['title']} (#{r['id']})" for r in defaults["always_on_rulebooks"]) or "none"
others = ", ".join(f"{r['title']} (#{r['id']})" for r in defaults["other_rulebooks"]) or "none"
books = ", ".join(f"{r['title']} (#{r['id']})" for r in defaults["rulebooks"]) or "none"
designs = ", ".join(f"{d['title']} (#{d['id']})" for d in defaults["design_systems"]) or "none"
return {
"defaults": defaults,
"ask": (
"This project has no inception decision: nobody has said what it "
f"inherits. Today, by default: always-on rulebooks binding it — {always}; "
f"rulebooks it could subscribe to — {others}; design system — "
f"inherits. Rulebooks it could subscribe to — {books}; design system — "
f"{'#' + str(defaults['design_system_id']) if defaults['design_system_id'] else 'none'} "
f"(available: {designs}); Systems — {defaults['systems']}. Ask the operator, "
"once: which always-on rulebooks to EXCLUDE here (default: none), which "
"rulebooks to subscribe, which design system (or none), and whether to seed "
"once: which rulebooks to subscribe (default: none — a rulebook binds "
"a project only when it opts in), which design system (or none), and "
"whether to seed "
"the standard starter Systems — then record the answers. This ask repeats on "
"every enter_project until a decision is recorded."
),
"call": (
f"decide_project_inception(project_id={project_id}, "
"exclude_always_on_rulebooks=[...], subscribe_rulebooks=[...], "
"subscribe_rulebooks=[...], "
"design_system_id=<id | -1 for none>, seed_systems=<true|false>)"
),
}
src/scribe/services/plugin_context.py
+37 -110
View File
@@ -8,7 +8,7 @@ Design note — altitude: we inject rule *titles* grouped by topic (a compact
index), NOT every rule's full statement. The 48 always-on statements run well
past the 10k-char `additionalContext` cap, and the push channel's job is to make
Claude *aware* the rules exist and *reach* for them — not to dump them. Full
text stays one `get_rule(id)` / `list_always_on_rules()` call away. Titles are
text stays one `get_rule(id)` / `search(content_type="rule")` call away. Titles are
mostly self-describing ("`dev` is home", "No GitHub — Fabled-Git only"), so the
index alone already steers behavior.
"""
@@ -1400,7 +1400,6 @@ async def build_write_path_hint(
repo_key: str = "",
exclude_derive: list[str] | None = None,
exclude_rule_ids: list[int] | None = None,
rules_etag: str = "",
) -> dict:
"""Prior-art hint for the plugin's PreToolUse hook on Write/Edit.
@@ -1686,60 +1685,22 @@ async def build_write_path_hint(
staleness: list[str] = []
# ── Have the rules moved under this session? (milestone 323) ───────
#
# THE CARRIER IS THE POINT. This hook already fires before a write — the
# moment acting on a stale rule actually costs something — and the check
# is one comparison against a marker the session already holds. No
# payload, no extra round trip, and nothing said when nothing moved.
# THE RULES-ETAG STALENESS ARM IS GONE (milestone 394).
#
# WHAT THIS CANNOT SEE, and a reader who finds an etag here will assume
# otherwise:
# It took a marker the session had been given at SessionStart, compared it
# against the resident set as it stood now, and said which rules had moved
# or fallen out of force. That was worth doing while a session held a
# fixed set of rules from turn zero and could be holding a stale copy of
# it hours later.
#
# what goes wrong | caught?
# ---------------------------------------------------|--------
# another session edits a rule mid-flight | yes
# the session is misremembering a rule read hours ago | yes
# compaction summarised the rules out of context | NO
# Nothing is resident now. A rule is retrieved at the moment it applies,
# so a session cannot be holding an out-of-date one — the next act that
# needs it fetches it again. The staleness this arm reported was an
# artifact of the delivery model rather than a fact about the corpus, and
# it goes with the model.
#
# The third is the most common and this is blind to it: the etag was in
# context too and went with the rules. The SessionStart nudge is that
# case's only mechanism and must not be softened because this shipped.
#
# Fails open, like every other arm here: a staleness hint must never
# break a write.
if rules_etag:
try:
current = await rulebooks_svc.list_always_on_rules(
user_id, project_id=project_id or 0,
)
if rulebooks_svc.rules_etag(current) != rules_etag:
moved = rulebooks_svc.rules_moved_since(current, rules_etag)
held = rulebooks_svc.etag_count(rules_etag)
bits = []
if moved:
named = ", ".join(
f"#{r.id} \u201c{r.title}\u201d" for r in moved[:3]
)
more = len(moved) - 3
bits.append(
f"{named}" + (f", and {more} more" if more > 0 else "")
)
# A DELETED rule moves no timestamp and leaves no row to name,
# so the count is the only thing that can report the one change
# that takes an instruction OUT of force.
if held is not None and held != len(current):
delta = len(current) - held
bits.append(
f"{abs(delta)} rule(s) {'added' if delta > 0 else 'no longer in force'}"
)
if bits:
staleness.append(
"Your loaded rules have changed since this session "
"started — " + "; ".join(bits) + ". Re-read them with "
"list_always_on_rules() before relying on the set you "
"are holding."
)
except Exception:
logger.debug("write-path rules-etag arm failed", exc_info=True)
# `staleness` survives as the list the arms below still append to.
# The guard sits BELOW the staleness arm on purpose. A rules change is
# unconditional news — it does not become less true because this
@@ -2208,67 +2169,37 @@ async def build_session_context(
its normalized key — triggers a one-line "bind this repo" hint so
the binding is self-healing.
Returns {"context": str, "rule_count": int, "project": dict | None,
"rules_etag": str}. The etag is for the HOOK, not for the model — the
hook stores it and hands it back on each write so the server can say
whether these rules have moved since the session loaded them.
Returns {"context": str, "project": dict | None}.
It carried `rule_count` and `rules_etag` until milestone 394, when the
preload it described was removed. The etag let the hook hand a marker back
on each write so the server could say whether the resident rules had
moved; nothing is resident now, so nothing can have moved, and a rule is
re-retrieved at the moment it applies rather than held and aged.
`context` is markdown ready to drop into `additionalContext`; it is capped
at _MAX_CHARS with an explicit truncation note so the hook can pass it
through verbatim.
"""
# Inside a project, the always-on set is the project's: an inception
# exclusion (milestone 297) takes a rulebook out of this block, and is
# named below so the departure is visible rather than silent.
rules = await rulebooks_svc.list_always_on_rules(user_id, project_id=project_id)
# AMBIENT source, and the one that matters most: this is the preload — the
# block every session opens with, chosen by nobody, paid for every turn.
#
# It emitted nothing until 2026-09-03, which made the resident set's cost
# certain and its usefulness unfalsifiable at the same time (#3473). Note
# #3089 is the argument this measurement finally lets someone test: that a
# rule arriving with thirty others, none of them relevant, is read as
# preamble rather than as a claim — so presence is not surfacing, and a
# tier-1 set can grow without anybody noticing it stopped working.
#
# Recorded even when the hook truncates the block below: the rules WERE
# delivered, and counting only the untruncated ones would quietly shrink
# the denominator exactly where the set is too big to read.
record_rule_surfaced(
user_id=user_id,
rule_ids=[r.id for r in rules],
source="session_start",
)
excluded = (
await rulebooks_svc.excluded_always_on_rulebooks(user_id, project_id)
if project_id else []
)
topic_map = await _topic_titles({r.topic_id for r in rules if r.topic_id})
lines: list[str] = [
"# Scribe — standing session context (auto-injected by the Scribe plugin)",
"",
"You are working with Scribe, the operator's self-hosted second brain. "
"The always-on rules below are BINDING this session. Titles only — full "
"text via `list_always_on_rules()` or `get_rule(id)`.",
"You are working with Scribe, the operator's self-hosted second brain.",
"",
"## Always-on rules (by topic)",
"## You are not holding the operator's rules",
"",
"No rule has been loaded into this session, and that is deliberate. "
"Rules arrive when something you are about to do makes one relevant — "
"a command you are about to run, code you are writing, or what the "
"operator just asked for. On most turns none will, and that is the "
"surface working rather than failing.",
"",
"**\"No rule arrived\" means \"nothing matched\" — never \"there is no "
"rule.\"** Before a consequential act, one that is hard to reverse or "
"outward-facing, `search(content_type=\"rule\")` is how you ask. "
"Retrieval runs on its own and is a convenience; asking is what you do "
"when it matters and nothing has spoken.",
]
# rules already arrive ordered by rulebook/topic/order, so grouping by
# consecutive topic_id preserves the intended sequence.
current_topic: int | None = object() # sentinel distinct from any id/None
for r in rules:
if r.topic_id != current_topic:
current_topic = r.topic_id
heading = topic_map.get(r.topic_id, "ungrouped") if r.topic_id else "ungrouped"
lines.append(f"### {heading}")
lines.append(f"- [{r.id}] {r.title}")
if excluded:
names = ", ".join(f"{e['title']} (#{e['id']})" for e in excluded)
lines += [
"",
f"Excluded for this project by its inception decision (not binding here): {names}.",
]
project_dict: dict | None = None
if project_id:
@@ -2336,14 +2267,10 @@ async def build_session_context(
context = "\n".join(line for line in lines if line is not None)
if len(context) > _MAX_CHARS:
context = context[:_MAX_CHARS].rstrip() + "\n\n…(truncated — call list_always_on_rules())"
context = context[:_MAX_CHARS].rstrip() + \
"\n\n…(truncated — ask with search(content_type=\"rule\"))"
return {
"context": context,
"rule_count": len(rules),
"project": project_dict,
# Computed from the rules THIS payload was built from, not re-queried:
# the marker has to describe the set the session is actually holding,
# and a second query could disagree with the first.
"rules_etag": rulebooks_svc.rules_etag(rules),
}
src/scribe/services/retrieval_telemetry.py
+1 -1
View File
@@ -852,7 +852,7 @@ async def retrieval_summary(
# something else.
#
# `ambient` now carries the bulk deliveries — the SessionStart preload,
# `list_always_on_rules`, and every `rules_payload` surface (#3473). Before
# and every `rules_payload` surface (#3473). Before
# they emitted, this block had no ambient key and said the absence was a
# fact about the data. It was, and it was also the thing that made the
# always-on set impossible to judge: the largest rule surface in the
src/scribe/services/rule_usage.py
+3 -3
View File
@@ -46,7 +46,7 @@ AMBIENT VS RANKED. The note twin splits ranked surfacings from ambient ones
because `enter_project` and the skill sync put records in front of the agent
without choosing them, and counting those as surfacings makes recency read as
popularity (#2477). Rules have exactly that shape: the SessionStart preload,
`list_always_on_rules`, and every `rules_payload` surface hand over the whole
and every `rules_payload` surface hand over the whole
applicable set at once, chosen by nobody.
Until 2026-09-03 those bulk surfaces emitted nothing, and this module said so —
@@ -184,8 +184,8 @@ def record_rule_surfaced(
def record_rule_pulled(*, user_id: int | None, rule_id: int, source: str) -> None:
"""Fire-and-forget: record that a rule was opened in full.
A PULL is somebody choosing to open one record. `list_always_on_rules` and
`enter_project` are NOT pulls — they are bulk resident loads that hand over
A PULL is somebody choosing to open one record. `enter_project` is NOT a
pull — they are bulk resident loads that hand over
every applicable rule at once, and counting them would swamp the signal
with the very ambient delivery the ratio exists to distinguish from.
"""
src/scribe/services/rule_versions.py
+1 -1
View File
@@ -36,7 +36,7 @@ from scribe.models.rule_version import RuleVersion
# snapshots would bury the edits somebody is actually looking for.
SNAPSHOT_FIELDS = (
"title", "statement", "why", "how_to_apply", "when_to_apply",
"tier", "kind", "verify_with", "expires_when",
"kind", "verify_with", "expires_when",
)
src/scribe/services/rulebooks.py
+35 -263
View File
@@ -12,7 +12,7 @@ from collections.abc import Iterable
from datetime import datetime
from typing import Optional
from sqlalchemy import and_, delete as sql_delete, insert, or_, select
from sqlalchemy import and_, delete as sql_delete, false as sa_false, insert, or_, select
from scribe.models import async_session
from scribe.models.system import System
@@ -82,7 +82,7 @@ async def update_rulebook(
rb = result.scalar_one_or_none()
if rb is None:
return None
allowed = {"title", "description", "always_on"}
allowed = {"title", "description"}
for key, value in fields.items():
if key in allowed and value is not None:
setattr(rb, key, value)
@@ -293,7 +293,6 @@ async def _assert_rulebook_rule_owned(session, rule_id: int, user_id: int) -> No
# The vocabularies migration 0088's CHECK constraints enforce. Named here so
# a caller can be corrected before the database refuses it (rule 36 keeps the
# two in step; this keeps the error readable).
TIERS = ("always_on", "conditional")
RELATION_KINDS = ("co_surfaces", "overrides", "elaborates")
# Migration 0098's CHECK. `rule` binds; `preference` is how the operator
# wants work done — see the model comment for why both live on one table.
@@ -311,21 +310,11 @@ NULLABLE_RULE_TEXT = (
)
def _valid_tier(tier: str) -> str:
"""An unrecognised tier falls back to always_on — the SAFE direction.
Getting this wrong the other way would silently stop a rule binding, which
is the one failure this whole milestone exists to prevent. A rule that
preloads when it did not need to costs context; a rule that quietly stops
preloading costs the behaviour it was written for.
"""
return tier if tier in TIERS else "always_on"
def _valid_kind(kind: str) -> str:
"""An unrecognised kind falls back to `rule` — the SAFE direction.
Same shape as _valid_tier and the same argument, pointed at force instead
The unrecognised value falls back to the binding one — the SAFE
direction, pointed at force instead
of delivery. A preference wrongly treated as binding costs a little
friction: the reader is told something is required that was only
preferred. A rule wrongly treated as a preference costs the thing the rule
@@ -369,7 +358,6 @@ def rule_brief(rule: Rule, **extra) -> dict:
"title": rule.title,
"statement": rule.statement,
"topic_id": rule.topic_id,
"tier": rule.tier,
# Unconditional, and the payload cost is accepted deliberately. Every
# other optional key below is attached only when present, because an
# absent key should never read as a capability the record lacks. Force
@@ -503,7 +491,7 @@ async def rule_detail(user_id: int, rule: Rule, system_ids: list[int] | None = N
async def create_rule(
topic_id: int, user_id: int, title: str, statement: str,
why: str = "", how_to_apply: str = "", order_index: int = 0,
when_to_apply: str = "", tier: str = "always_on", arose_from_id: int = 0,
when_to_apply: str = "", arose_from_id: int = 0,
verify_with: str = "", expires_when: str = "", kind: str = "rule",
) -> Rule:
async with async_session() as session:
@@ -513,7 +501,6 @@ async def create_rule(
title=title,
statement=statement,
when_to_apply=when_to_apply or None,
tier=_valid_tier(tier),
kind=_valid_kind(kind),
why=why or None,
how_to_apply=how_to_apply or None,
@@ -532,7 +519,7 @@ async def create_rule(
async def create_project_rule(
project_id: int, user_id: int, title: str, statement: str,
why: str = "", how_to_apply: str = "", order_index: int = 0,
when_to_apply: str = "", tier: str = "always_on", arose_from_id: int = 0,
when_to_apply: str = "", arose_from_id: int = 0,
verify_with: str = "", expires_when: str = "", kind: str = "rule",
) -> Rule:
"""Create a rule scoped to a single project (no rulebook ceremony).
@@ -548,7 +535,6 @@ async def create_project_rule(
title=title,
statement=statement,
when_to_apply=when_to_apply or None,
tier=_valid_tier(tier),
kind=_valid_kind(kind),
why=why or None,
how_to_apply=how_to_apply or None,
@@ -632,89 +618,6 @@ async def list_rules(
return rulebook_rules + list(proj_result.scalars().all())
def _excluded_rulebook_ids_q(project_id: int):
"""Subquery: the always-on rulebooks this project opted out of at
inception (milestone 297) — used by every rule-resolution path so an
exclusion is total, not just cosmetic."""
from scribe.models.rulebook import project_rulebook_exclusions
return select(project_rulebook_exclusions.c.rulebook_id).where(
project_rulebook_exclusions.c.project_id == project_id
)
async def excluded_always_on_rulebooks(user_id: int, project_id: int) -> list[dict]:
"""[{id, title}] of the always-on rulebooks excluded for ``project_id``
(owner-scoped). Empty for an undecided or inherit-all project."""
from scribe.models.rulebook import project_rulebook_exclusions
if not project_id:
return []
async with async_session() as session:
rows = (
await session.execute(
select(Rulebook.id, Rulebook.title)
.join(project_rulebook_exclusions,
project_rulebook_exclusions.c.rulebook_id == Rulebook.id)
.where(
project_rulebook_exclusions.c.project_id == project_id,
Rulebook.owner_user_id == user_id,
Rulebook.deleted_at.is_(None),
)
.order_by(Rulebook.title)
)
).all()
return [{"id": rid, "title": title} for rid, title in rows]
async def list_always_on_rules(
user_id: int, limit: int = 100, project_id: int = 0,
) -> list[Rule]:
"""Return all rules from rulebooks flagged always_on for the user.
Called by the MCP tool of the same name at session start to load the
standing rules that apply regardless of which project (if any) is in
scope. Ordering matches list_rules so results are stable across calls.
``project_id`` (milestone 297): inside a project that excluded specific
always-on rulebooks at inception, those rulebooks' rules are NOT
returned — the project decided not to inherit them. 0 = the user-wide
set, which is what a session sees before a project is in scope.
"""
async with async_session() as session:
q = (
select(Rule)
.join(RulebookTopic, Rule.topic_id == RulebookTopic.id)
.join(Rulebook, RulebookTopic.rulebook_id == Rulebook.id)
.where(
Rulebook.owner_user_id == user_id,
Rulebook.always_on.is_(True),
Rule.deleted_at.is_(None),
RulebookTopic.deleted_at.is_(None),
Rulebook.deleted_at.is_(None),
# TIER (milestone 307). This is the SESSION-START call, made
# before any project is in scope — there is no area vocabulary
# to match a conditional rule against yet, so only the
# unconditional tier belongs here. A conditional rule reaches a
# session through enter_project (by area) or search (by
# meaning), not by being resident.
#
# Behaviour is unchanged until rules are actually re-tiered:
# `tier` defaults to always_on, so every existing rule still
# arrives exactly as it did.
Rule.tier == "always_on",
)
)
if project_id:
q = q.where(Rulebook.id.notin_(_excluded_rulebook_ids_q(project_id)))
result = await session.execute(
q.order_by(
Rulebook.id, RulebookTopic.order_index, Rule.order_index, Rule.title,
).limit(limit)
)
return list(result.scalars().all())
async def _fetch_owned_rule(session, rule_id: int, user_id: int) -> Optional[Rule]:
"""Fetch a rule by id, scoped to user owning either its rulebook
(via topic) or its project (via project_id). Honors soft-delete.
@@ -780,7 +683,7 @@ async def update_rule(
return None
allowed = {
"title", "statement", "why", "how_to_apply", "order_index",
"when_to_apply", "tier", "kind", "arose_from_id",
"when_to_apply", "kind", "arose_from_id",
"verify_with", "expires_when",
}
check_before = rule.verify_with
@@ -796,9 +699,7 @@ async def update_rule(
for key, value in fields.items():
if key not in allowed or value is None:
continue
if key == "tier":
value = _valid_tier(value)
elif key == "kind":
if key == "kind":
value = _valid_kind(value)
elif key in NULLABLE_RULE_TEXT:
value = value or None
@@ -808,9 +709,8 @@ async def update_rule(
# A verification stamp certifies A CHECK, not a rule. Rewrite or
# remove the check and the old stamp certifies something that no
# longer exists — so it is dropped, and the rule re-enters the sweep.
# The safe direction, for the same reason _valid_tier falls back to
# always_on: a rule wrongly listed as due costs one look, a rule
# wrongly vouched for costs the thing the sweep exists to catch.
# The safe direction: a rule wrongly listed as due costs one look, a
# rule wrongly vouched for costs the thing the sweep exists to catch.
if rule.verify_with != check_before:
rule.verified_at = None
# Same session as the edit, so the two commit together. The snapshot
@@ -1113,51 +1013,6 @@ async def unsuppress_rule_for_project(
await session.commit()
async def exclude_always_on_rulebook_for_project(
project_id: int, rulebook_id: int, user_id: int,
) -> None:
"""Opt one project out of a whole ALWAYS-ON rulebook (milestone 297).
Owner-only on both sides; the rulebook must be always_on — a subscribed
rulebook is left by unsubscribing, not excluding. Idempotent."""
from scribe.models.rulebook import project_rulebook_exclusions
async with async_session() as session:
await _assert_project_owned(session, project_id, user_id)
await _assert_rulebook_owned(session, rulebook_id, user_id)
rb = await session.get(Rulebook, rulebook_id)
if rb is None or not rb.always_on:
raise ValueError(
f"rulebook {rulebook_id} is not always-on — it binds only by "
"subscription; unsubscribe_project_from_rulebook instead"
)
try:
await session.execute(
insert(project_rulebook_exclusions).values(
project_id=project_id, rulebook_id=rulebook_id,
)
)
await session.commit()
except IntegrityError:
await session.rollback() # already excluded — idempotent
async def include_always_on_rulebook_for_project(
project_id: int, rulebook_id: int, user_id: int,
) -> None:
"""Undo exclude_always_on_rulebook_for_project. Idempotent."""
from scribe.models.rulebook import project_rulebook_exclusions
async with async_session() as session:
await _assert_project_owned(session, project_id, user_id)
await session.execute(
sql_delete(project_rulebook_exclusions).where(
project_rulebook_exclusions.c.project_id == project_id,
project_rulebook_exclusions.c.rulebook_id == rulebook_id,
)
)
await session.commit()
async def suppress_topic_for_project(
project_id: int, topic_id: int, user_id: int,
) -> None:
@@ -1326,7 +1181,6 @@ async def get_applicable_rules(
Rulebook.deleted_at.is_(None),
# An inception exclusion is total (milestone 297): a rulebook the
# project opted out of contributes nothing, subscribed or not.
Rulebook.id.notin_(_excluded_rulebook_ids_q(project_id)),
)
.order_by(
Rulebook.id, RulebookTopic.order_index, Rule.order_index, Rule.title,
@@ -1337,11 +1191,10 @@ async def get_applicable_rules(
rules_q = rules_q.where(Rule.id.notin_(suppressed_rule_ids))
if suppressed_topic_ids:
rules_q = rules_q.where(Rule.topic_id.notin_(suppressed_topic_ids))
# TIER (milestone 307). always_on rules are resident, as every rule was
# before tiers existed. A conditional rule is REACHABLE, and reaches
# this project only when it is tagged to an area this project actually
# works in — a deterministic tag match, never a similarity score, so
# bindingness never depends on a ranking (D7).
# AREA BINDING (milestone 307, narrowed by 394). A rule reaches this
# project when it is tagged to an area the project actually works in —
# a deterministic tag match, never a similarity score, so bindingness
# never depends on a ranking (D7).
#
# Applied in SQL rather than by filtering afterwards, so `limit` counts
# the rules that will actually be surfaced instead of counting rules
@@ -1357,10 +1210,21 @@ async def get_applicable_rules(
reachable = select(rule_systems.c.rule_id).where(
rule_systems.c.canonical_id.in_(project_area_ids)
) if project_area_ids else None
tier_clause = (Rule.tier == "always_on")
if reachable is not None:
tier_clause = or_(tier_clause, Rule.id.in_(reachable))
rules_q = rules_q.where(tier_clause)
# AREA REACHABILITY IS NOW THE WHOLE TEST (milestone 394). This read
# `always_on OR reachable`, so a subscribed rulebook's resident rules
# arrived here whatever the project did. The tier is gone, and
# dropping its arm rather than the whole clause is the deliberate
# half: what survives is the DETERMINISTIC one — a rule binds this
# project because it is tagged to an area the project actually works
# in (D7), never because a similarity score cleared a bar.
#
# A project with no canonical-tagged Systems therefore gets no bulk
# rules here, and that is the reading rather than a gap: rules still
# reach it by retrieval, when something it is doing makes one
# relevant. Handing over every subscribed rule instead would make this
# payload BIGGER than the preload this milestone exists to remove.
rules_q = (rules_q.where(Rule.id.in_(reachable)) if reachable is not None
else rules_q.where(sa_false()))
rule_rows = (await session.execute(rules_q)).all()
truncated = len(rule_rows) > limit
rules = [
@@ -1381,12 +1245,11 @@ async def get_applicable_rules(
)
.order_by(Rule.order_index, Rule.title)
)
if reachable is not None:
proj_rules_q = proj_rules_q.where(
or_(Rule.tier == "always_on", Rule.id.in_(reachable))
)
else:
proj_rules_q = proj_rules_q.where(Rule.tier == "always_on")
# A PROJECT'S OWN RULES ARE NOT FILTERED BY AREA, and the asymmetry
# with the family query above is the point. A family rule has to earn
# its way into this project; a rule written ON this project is scoped
# to it by construction, and filtering it again would drop rules whose
# only fault is that nobody tagged them to a System.
proj_rule_rows = (await session.execute(proj_rules_q)).all()
project_rules = [rule_brief(rule) for (rule,) in proj_rule_rows]
@@ -1422,7 +1285,6 @@ async def get_applicable_rules(
"suppressed_topics": suppressed_topics,
"truncated": truncated,
"subscribed_rulebooks": subscribed_rulebooks,
"excluded_always_on": await excluded_always_on_rulebooks(user_id, project_id),
}
@@ -1434,9 +1296,6 @@ def rules_payload(applicable: dict, *, user_id: int | None, source: str) -> dict
same seven keys under the same names — so a reader learns them once. One
place renames `rules` → `applicable_rules` and `truncated` →
`applicable_rules_truncated`; the tools merge this into their payloads.
`excluded_always_on` (milestone 297) names the always-on rulebooks this
project decided NOT to inherit, so the departure is visible wherever the
rules are.
IT ALSO RECORDS THE SURFACING, which is why it now takes a caller and a
source. Every one of those surfaces is a bulk delivery — the applicable set
@@ -1453,7 +1312,7 @@ def rules_payload(applicable: dict, *, user_id: int | None, source: str) -> dict
Emitting from here is safe in a way emitting from `get_applicable_rules`
would not be: this function is only ever called to BUILD A REPLY. The two
other callers of the rules machinery — the write-path etag arm
(`plugin_context`) and `rules_etag_for` — compute a marker and show nobody
(`plugin_context`) — computed a marker and showed nobody
anything, and counting those would put rules in the denominator that no
agent ever saw.
"""
@@ -1472,7 +1331,6 @@ def rules_payload(applicable: dict, *, user_id: int | None, source: str) -> dict
"project_rules": applicable.get("project_rules", []),
"suppressed_rules": applicable.get("suppressed_rules", []),
"suppressed_topics": applicable.get("suppressed_topics", []),
"excluded_always_on": applicable.get("excluded_always_on", []),
}
@@ -1497,86 +1355,11 @@ def rules_payload(applicable: dict, *, user_id: int | None, source: str) -> dict
_ETAG_EMPTY = "empty|0"
def rules_etag(rules: list) -> str:
"""A marker for "is the set you are holding still the current one?".
`max(updated_at)` alone is not enough: DELETING a rule moves no timestamp,
and that is the single change that takes an instruction OUT of force —
the one a session most needs to hear about. The count catches it.
Instance-agnostic (rule 115): it knows nothing about any particular
rulebook, and an install with one rule or none produces a stable marker
rather than an error. "No rules" must read as a state, not as a change,
or every session on a fresh install would be told its rules had moved.
"""
if not rules:
return _ETAG_EMPTY
# A decoration must not be able to break what it decorates. This is
# computed on the SessionStart path, where raising would cost the whole
# context payload to save a hint — so a row with no usable timestamp is
# skipped rather than compared, and a set with none degrades to a
# count-only marker instead of failing. Count-only still catches a rule
# added or deleted; it just cannot see an edit, which is the right way
# round to lose information.
stamps = [
r.updated_at for r in rules
if isinstance(getattr(r, "updated_at", None), datetime)
]
if not stamps:
return f"unknown|{len(rules)}"
return f"{max(stamps).isoformat()}|{len(rules)}"
async def rules_etag_for(user_id: int, project_id: int = 0) -> str:
"""The current marker for the set a session at this scope would hold.
Deliberately built from `list_always_on_rules` rather than from a
`max()/count()` aggregate. An aggregate would be cheaper, and would have
to restate that function's definition of the set — the always_on flag,
the project's inception exclusions, the tier filter. Two definitions of
"the session's rules" is how the marker starts disagreeing with the
rules, which is worse than materialising a few dozen rows.
"""
rules = await list_always_on_rules(user_id, project_id=project_id)
return rules_etag(rules)
def rules_moved_since(rules: list, held_etag: str) -> list:
"""The rules whose text changed after `held_etag` was issued.
Returns [] when the marker matches, is unparseable, or is absent — a
caller cannot act on "something is different but I cannot say what", and
a garbled marker must not be reported as a change.
A count difference is real news that this list cannot show: a rule
DELETED since the marker was issued has no row left to return. Callers
compare counts separately.
"""
if not held_etag or held_etag == _ETAG_EMPTY:
return []
stamp, _, _count = held_etag.partition("|")
try:
held_at = datetime.fromisoformat(stamp)
except ValueError:
return []
return [r for r in rules if r.updated_at and r.updated_at > held_at]
def etag_count(held_etag: str) -> int | None:
"""How many rules the holder had. None when the marker cannot be read."""
_stamp, _, count = (held_etag or "").partition("|")
try:
return int(count)
except ValueError:
return None
# ── The staleness sweep (milestone 312) ────────────────────────────────
async def rules_due_for_verification(
user_id: int,
older_than_days: int = 0,
tier: str = "",
never_only: bool = False,
) -> list[Rule]:
"""Rules that carry a check, oldest verification first, never-checked top.
@@ -1605,20 +1388,12 @@ async def rules_due_for_verification(
older_than_days: only rules last verified longer ago than this.
Never-checked rules always qualify — they are the most overdue
thing there is. 0 = no age filter.
tier: "always_on" or "conditional" to narrow. Raises on anything else
rather than falling back: _valid_tier's silent always_on default
is right for a WRITE (the safe direction is to keep binding), and
wrong for a FILTER, where it would quietly answer a different
question than the one asked.
never_only: only rules that have never been verified.
"""
from datetime import datetime, timedelta, timezone
from scribe.models.project import Project
if tier and tier not in TIERS:
raise ValueError(f"tier must be one of {TIERS}, got {tier!r}")
async with async_session() as session:
stmt = (
select(Rule)
@@ -1641,8 +1416,6 @@ async def rules_due_for_verification(
),
)
)
if tier:
stmt = stmt.where(Rule.tier == tier)
if never_only:
stmt = stmt.where(Rule.verified_at.is_(None))
elif older_than_days > 0:
@@ -1668,7 +1441,6 @@ def verification_row(rule: Rule) -> dict:
"id": rule.id,
"title": rule.title,
"statement": rule.statement,
"tier": rule.tier,
"topic_id": rule.topic_id,
"project_id": rule.project_id,
"when_to_apply": rule.when_to_apply or "",