diff --git a/alembic/versions/0099_scrub_secrets_from_retrieval_logs.py b/alembic/versions/0099_scrub_secrets_from_retrieval_logs.py new file mode 100644 index 0000000..c643d21 --- /dev/null +++ b/alembic/versions/0099_scrub_secrets_from_retrieval_logs.py @@ -0,0 +1,104 @@ +"""scrub credential-shaped spans out of retrieval_logs.query + +Revision ID: 0099 +Revises: 0098 +Create Date: 2026-09-11 + +`pre_tool_rule` retrieves against the RAW COMMAND TEXT and `write_path_rule` +against the code being written, so whatever was on the command line or in the +buffer is what `record_retrieval` wrote into `retrieval_logs.query`. A command +that exported a token stored the token (#3925). + +`services/retrieval_telemetry.scrub_secrets` closes that going forward — the +value never reaches the column. It cannot reach BACKWARDS, and this does: it +rewrites the rows already written. + +REDACTED IN PLACE, NOT DELETED. The rest of the row — score, threshold, +result count, duration, the near-miss record id — is legitimate evidence, and +it is what a threshold is tuned from. Deleting the row would throw that away +to remove a secret that lives in one column, so the column is what gets +rewritten. Rows with no credential in them are not touched at all. + +THE PATTERNS ARE INLINED RATHER THAN IMPORTED, deliberately, against the DRY +instinct. A migration is a frozen record of a change that already happened on +every install that ran it; importing the live patterns would mean this +migration quietly does something different next year than it did when it ran, +and two installs at the same revision would no longer be in the same state. +The Python twin in `services/retrieval_telemetry.py` is free to grow — this is +what ran here, once. The one thing that must not drift is coverage, and the +guard for that is `test_retrieval_query_scrubbing.py`, which tests the live +function rather than this copy. + +POSIX regex, not Python's. Postgres ARE supports the non-greedy `*?` the PEM +pattern needs, and `\\s`/`\\S`, so the shapes port directly. The `'gi'` flags +are global + case-insensitive, matching `re.sub` with `(?i)`. + +NO BARE `auth` IN THE ASSIGNED PATTERN. It matches `--author=`, so a commit +naming an address would have had the address redacted — evidence eaten for a +word that only looks credential-shaped. `AUTH_TOKEN` is still caught, by +`token`. + +Downgrade is a no-op, and honestly so: the original text is gone and a +migration cannot invent it back. Saying that plainly is better than a +downgrade that appears to restore something and does not. +""" +from alembic import op + +revision = "0099" +down_revision = "0098" +branch_labels = None +depends_on = None + +# Vendor-prefixed credentials — the prefix IS the tell, so no entropy guessing. +_TOKEN = ( + r"(fmcp_|flt_|ghp_|gho_|ghs_|ghu_|github_pat_|glpat-|xox[abprs]-" + r"|sk-[A-Za-z0-9]*-?|AKIA|ASIA)[A-Za-z0-9_\-]{12,}" +) +# A value handed to a secret-NAMED variable, in shell, env, YAML, JSON or a +# query string. The NAME identifies it, so the value can be anything. +_ASSIGNED = ( + r"([A-Za-z0-9_]*(token|secret|password|passwd|api[_-]?key|access[_-]?key)" + r"[A-Za-z0-9_]*)(\s*[:=]\s*[\"']?)([^\s\"'&]{8,})" +) +_AUTH_HEADER = r"(authorization\s*:\s*(bearer|basic|token)\s+)(\S+)" +_PEM = ( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----(.|\n)*?-----END [A-Z ]*PRIVATE KEY-----" +) + + + +def _lit(pattern: str) -> str: + """A regex as a SQL string literal. + + A single quote inside a single-quoted SQL literal has to be DOUBLED, and + the assigned-value pattern contains two of them (it allows an optional + quote around the value). Left unescaped they close the literal early and + the migration dies on a syntax error — which is the whole reason this + helper exists rather than the patterns being pasted in inline. + """ + return pattern.replace("'", "''") + + +_SCRUB_SQL = f""" +UPDATE retrieval_logs +SET query = regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace(query, '{_lit(_TOKEN)}', '[redacted:token]', 'gi'), + '{_lit(_ASSIGNED)}', '\\1\\3[redacted:assigned]', 'gi'), + '{_lit(_AUTH_HEADER)}', '\\1[redacted:auth-header]', 'gi'), + '{_lit(_PEM)}', '[redacted:private-key]', 'gi') +WHERE query IS NOT NULL + AND (query ~* '{_lit(_TOKEN)}' + OR query ~* '{_lit(_ASSIGNED)}' + OR query ~* '{_lit(_AUTH_HEADER)}' + OR query ~* '{_lit(_PEM)}') +""" + + +def upgrade() -> None: + op.execute(_SCRUB_SQL) + + +def downgrade() -> None: + """Deliberately empty — the original text no longer exists to restore.""" diff --git a/alembic/versions/0100_drop_always_on_tier.py b/alembic/versions/0100_drop_always_on_tier.py new file mode 100644 index 0000000..a524281 --- /dev/null +++ b/alembic/versions/0100_drop_always_on_tier.py @@ -0,0 +1,96 @@ +"""drop the always-on tier: rules.tier, rulebooks.always_on, the exclusions table + +Revision ID: 0100 +Revises: 0099 +Create Date: 2026-09-11 + +Milestone 394. Every rule now reaches a session by retrieval — because +something it is about to do made the rule relevant — and the machinery that +delivered rules unconditionally goes with it. + +WHAT GOES, AND WHERE IT CAME FROM + + - ``rules.tier`` and its ``ck_rules_tier`` CHECK (migration 0088). Dropping + the column takes the constraint with it. Rule 36 is about ADDING a value to + a live whitelist, which needs DROP + ADD in the same migration; it does not + speak to removing the column outright, and saying so here is cheaper than + the next reader wondering whether it was forgotten. + - ``rule_versions.tier`` (migration 0098). A version records what a rule + SAID; with no tier on a rule there is nothing for a snapshot to carry. + - ``rulebooks.always_on`` (migration 0058). A rulebook reaches a project by + subscription now, and by nothing else. + - ``project_rulebook_exclusions`` (migration 0085). It recorded a project's + opt-out of an always-on rulebook. Opting out of something that no longer + binds you is not a state that can exist — declining a rulebook is + expressed by not subscribing to it. + +IRREVERSIBLE, AND THE DOWNGRADE SAYS SO RATHER THAN PRETENDING + +The downgrade recreates the columns and the table with their DEFAULTS. It +cannot restore WHICH rules were always-on, which rulebooks bound every project, +or which projects had opted out — that information is in what this drops. + +That distinction is the one this repo keeps insisting on: a value invented to +fill a hole is not a measurement. So a downgraded database is structurally able +to run the old code and is NOT the database the old code was running against — +every rule comes back at the ``always_on`` default, which for the tier column +happens to mean "binding", the safe direction to be wrong in. + +Anyone who needs the real prior state restores a backup taken before this ran. +""" +import sqlalchemy as sa +from alembic import op + +revision = "0100" +down_revision = "0099" +branch_labels = None +depends_on = None + +_TIERS = ("always_on", "conditional") + + +def _in_list(column: str, values: tuple[str, ...]) -> str: + return f"{column} IN (" + ", ".join(f"'{v}'" for v in values) + ")" + + +def upgrade() -> None: + op.drop_table("project_rulebook_exclusions") + op.drop_column("rulebooks", "always_on") + op.drop_column("rule_versions", "tier") + # The CHECK goes with the column it constrains; naming it here would be a + # second drop of the same object. + op.drop_column("rules", "tier") + + +def downgrade() -> None: + """Structure only. See the module docstring — the values are gone.""" + op.add_column( + "rules", + sa.Column("tier", sa.Text(), nullable=False, server_default="always_on"), + ) + op.create_check_constraint("ck_rules_tier", "rules", _in_list("tier", _TIERS)) + op.add_column("rule_versions", sa.Column("tier", sa.Text(), nullable=True)) + op.add_column( + "rulebooks", + sa.Column( + "always_on", sa.Boolean(), nullable=False, + server_default=sa.text("false"), + ), + ) + op.create_table( + "project_rulebook_exclusions", + sa.Column( + "project_id", sa.BigInteger(), + sa.ForeignKey("projects.id", ondelete="CASCADE"), + primary_key=True, nullable=False, + ), + sa.Column( + "rulebook_id", sa.BigInteger(), + sa.ForeignKey("rulebooks.id", ondelete="CASCADE"), + primary_key=True, nullable=False, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=True, + ), + ) diff --git a/docs/api-keys-and-mcp.md b/docs/api-keys-and-mcp.md index d950d4b..4157fe0 100644 --- a/docs/api-keys-and-mcp.md +++ b/docs/api-keys-and-mcp.md @@ -89,7 +89,7 @@ table here. The tools are grouped by family: | Projects / Milestones | `enter_project`, `get_project`, `create_milestone`, … | Containers and outcomes | | Search / Recall | `search`, `get_recent`, `list_tags`, `retrieval_telemetry` | Semantic + structured recall, and the readout its thresholds are tuned from | | Systems | `create_system`, `list_systems`, `list_system_records` | Reusable per-project subsystems/areas | -| Rulebooks | `list_always_on_rules`, `list_rules`, `create_rule`, `create_project_rule`, `subscribe_project_to_rulebook`, … | Engineering/workflow rules | +| Rulebooks | `list_rules`, `create_rule`, `create_project_rule`, `subscribe_project_to_rulebook`, … | Engineering/workflow rules | | Processes | `list_processes`, `get_process`, `create_process` | Saved prompts/workflows | | Trash | `list_trash`, `restore`, `purge_trash` | Recoverable deletes | | Admin | `get_app_logs` (write/admin key) | Diagnostics | diff --git a/docs/api-reference.md b/docs/api-reference.md index fbf7c0a..1679231 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -77,7 +77,7 @@ endpoint at `/mcp`, not these REST routes. |--------|------|-------------| | GET / POST | `/api/projects` | List (owned + shared) / create | | GET / PATCH / DELETE | `/api/projects/:id` | Read (with `milestone_summary`, `inception`) / update / delete | -| POST | `/api/projects/:id/inception` | Record what the project inherits `{choices: {exclude_always_on_rulebooks, subscribe_rulebooks, design_system_id, seed_systems}}` (owner-only; `POST /api/projects` accepts the same under `inception`) | +| POST | `/api/projects/:id/inception` | Record what the project inherits `{choices: {subscribe_rulebooks, design_system_id, seed_systems}}` (owner-only; `POST /api/projects` accepts the same under `inception`) | | GET | `/api/projects/:id/inception/defaults` | What binds if nobody decides — the inception card's payload | | GET | `/api/projects/:id/notes` | Notes + tasks in this project | | GET / POST | `/api/projects/:id/milestones` | List / create milestones | @@ -120,7 +120,6 @@ endpoint at `/mcp`, not these REST routes. | POST | `/api/projects/:id/rules` | Create a project-scoped rule | | POST / DELETE | `/api/projects/:id/suppressions/rules/:rid` | Suppress / unsuppress a rule | | POST / DELETE | `/api/projects/:id/suppressions/topics/:tid` | Suppress / unsuppress a topic | -| POST / DELETE | `/api/projects/:id/exclusions/rulebooks/:rid` | Exclude / include an always-on rulebook for this project (inception) | ## Sharing @@ -206,6 +205,6 @@ endpoint at `/mcp`, not these REST routes. Claude clients connect to the built-in MCP server at `POST /mcp` (streamable HTTP, Bearer auth with an `fmcp_` key), served by `src/scribe/mcp/`. It is not a REST surface — it exposes the same data as typed tools (`create_note`, `create_task`, -`start_planning`, `search`, `enter_project`, `list_always_on_rules`, …) with +`start_planning`, `search`, `enter_project`, …) with server-level usage guidance delivered in the MCP `instructions` block. See [API Keys & MCP](api-keys-and-mcp.md). diff --git a/docs/features.md b/docs/features.md index 694223a..d45fcfa 100644 --- a/docs/features.md +++ b/docs/features.md @@ -60,8 +60,10 @@ Scribe stores the operator's engineering and workflow **rules** so Claude follow across sessions. - **Rulebooks → topics → rules** — Rules are grouped by topic inside a rulebook. -- **Always-on rules** — A rulebook can be flagged always-on; its rules load at the - start of every session through the plugin's push channel. +- **Rules arrive by retrieval** — Nothing is preloaded. A rule reaches a + session when what the agent is about to do matches its trigger: a command, + a file being written, or the operator's own message. `when_to_apply` is + therefore the field that decides whether a rule is ever seen. - **Per-project scope** — A project subscribes to rulebooks, and can add project-scoped rules or suppress individual inherited rules/topics. @@ -94,7 +96,7 @@ The whole store is reachable by Claude through a built-in **MCP endpoint at `/mc (Bearer-auth with an API key). The **Scribe Claude Code plugin** (shipped in this repo) wires it up: -- a `SessionStart` hook that injects the operator's always-on rules + active-project +- a `SessionStart` hook that injects active-project context so Scribe surfaces without being asked (fail-open if Scribe is unreachable); - universal process-skills — writing-plans, systematic-debugging, verification, brainstorming — that route their output into Scribe; diff --git a/frontend/src/api/inception.ts b/frontend/src/api/inception.ts index 05773e1..0321c78 100644 --- a/frontend/src/api/inception.ts +++ b/frontend/src/api/inception.ts @@ -2,7 +2,6 @@ import { apiGet, apiPost } from "@/api/client"; export interface InceptionChoices { - exclude_always_on_rulebooks: number[]; subscribe_rulebooks: number[]; design_system_id: number | null; seed_systems: boolean; @@ -16,9 +15,7 @@ export interface InceptionRecord { } export interface InceptionDefaults { - always_on_rulebooks: { id: number; title: string }[]; - other_rulebooks: { id: number; title: string }[]; - excluded_always_on: { id: number; title: string }[]; + rulebooks: { id: number; title: string }[]; subscribed_rulebooks: { id: number; title: string }[]; design_system_id: number | null; design_systems: { id: number; title: string }[]; @@ -32,7 +29,7 @@ export interface InceptionDecision { } export const emptyChoices = (): InceptionChoices => ({ - exclude_always_on_rulebooks: [], subscribe_rulebooks: [], design_system_id: null, seed_systems: false, + subscribe_rulebooks: [], design_system_id: null, seed_systems: false, }); export const fetchInceptionDefaults = (projectId: number) => diff --git a/frontend/src/api/rulebooks.ts b/frontend/src/api/rulebooks.ts index 1a8c175..7b0fca1 100644 --- a/frontend/src/api/rulebooks.ts +++ b/frontend/src/api/rulebooks.ts @@ -3,7 +3,6 @@ import type { RecordUsage } from "@/types/usage"; import { apiGet, apiPost, apiPatch, apiDelete } from "@/api/client"; /** How a rule reaches a session (milestone 307). */ -export type RuleTier = "always_on" | "conditional"; /** * A typed edge between two rules. Each kind exists because its absence forced @@ -26,7 +25,6 @@ export interface Rulebook { owner_user_id: number; title: string; description: string; - always_on: boolean; created_at: string | null; updated_at: string | null; } @@ -49,12 +47,6 @@ export interface Rule { statement: string; /** WHEN this rule fires — the trigger, not the instruction. */ when_to_apply: string; - /** - * always_on preloads into every session; conditional is reachable and - * surfaced when its trigger fires. A rule with no tier set behaves as - * always_on, which is how every rule behaved before this existed. - */ - tier: RuleTier; why: string; how_to_apply: string; /** @@ -87,7 +79,6 @@ export interface RuleHeader { title: string; statement: string; topic_id: number | null; - tier: RuleTier; /** A date (YYYY-MM-DD), not a timestamp. */ updated_at: string | null; when_to_apply?: string; @@ -134,7 +125,6 @@ export interface ApplicableRules { truncated: boolean; subscribed_rulebooks: { id: number; title: string }[]; /** Always-on rulebooks this project opted out of at inception (milestone 297). */ - excluded_always_on: { id: number; title: string }[]; } // ── Rulebooks ─────────────────────────────────────────────────────── @@ -152,7 +142,7 @@ export async function createRulebook(data: { title: string; description?: string return apiPost("/api/rulebooks", data); } -export async function updateRulebook(id: number, data: Partial<{ title: string; description: string; always_on: boolean }>): Promise { +export async function updateRulebook(id: number, data: Partial<{ title: string; description: string }>): Promise { return apiPatch(`/api/rulebooks/${id}`, data); } @@ -207,7 +197,6 @@ export interface RuleWrite { title: string; statement: string; when_to_apply: string; - tier: RuleTier; why: string; how_to_apply: string; order_index: number; @@ -258,7 +247,6 @@ export interface RuleVersion { why?: string; how_to_apply?: string; when_to_apply?: string; - tier?: string; verify_with?: string; expires_when?: string; } @@ -323,16 +311,6 @@ export async function unsuppressTopicForProject(projectId: number, topicId: numb return apiDelete(`/api/projects/${projectId}/suppressions/topics/${topicId}`); } -// ── Always-on exclusions (milestone 297) ──────────────────────────────────── - -export async function excludeAlwaysOnRulebook(projectId: number, rulebookId: number): Promise { - await apiPost(`/api/projects/${projectId}/exclusions/rulebooks/${rulebookId}`, {}); -} - -export async function includeAlwaysOnRulebook(projectId: number, rulebookId: number): Promise { - await apiDelete(`/api/projects/${projectId}/exclusions/rulebooks/${rulebookId}`); -} - /** * One row of the staleness sweep. Unlike RuleHeader this carries the CHECK @@ -343,7 +321,6 @@ export interface RuleVerificationRow { id: number; title: string; statement: string; - tier: RuleTier; topic_id: number | null; project_id: number | null; when_to_apply: string; @@ -366,12 +343,10 @@ export interface RuleVerificationRow { */ export async function listRulesDueForVerification(opts: { olderThanDays?: number; - tier?: RuleTier; neverOnly?: boolean; } = {}): Promise<{ rules: RuleVerificationRow[]; total: number }> { const q = new URLSearchParams(); if (opts.olderThanDays) q.set("older_than_days", String(opts.olderThanDays)); - if (opts.tier) q.set("tier", opts.tier); if (opts.neverOnly) q.set("never_only", "true"); const qs = q.toString(); return apiGet(`/api/rules-due-for-verification${qs ? `?${qs}` : ""}`); diff --git a/frontend/src/assets/rules-shared.css b/frontend/src/assets/rules-shared.css index 72c1448..f2345f3 100644 --- a/frontend/src/assets/rules-shared.css +++ b/frontend/src/assets/rules-shared.css @@ -20,6 +20,15 @@ milestone (tier, then verification) and were byte-identical; a third would have drifted. The pane's italic serif title is inherited by anything inside it, so the chip resets family and style explicitly. */ +/* A rule with no trigger cannot be retrieved, and since milestone 394 + retrieval is the only delivery — so this marks a rule that will never + reach a session. Warning rather than error: the rule is not broken, it is + unreachable, and the fix is one field away. */ +.rule-chip-inert { + color: var(--fs-warning-fg); + background: color-mix(in srgb, var(--fs-warning) 12%, var(--fs-surface-raised)); +} + .rule-chip { margin-left: 0.4rem; font-family: var(--fs-font-body); diff --git a/frontend/src/components/InceptionCard.vue b/frontend/src/components/InceptionCard.vue index e33f842..f775f6f 100644 --- a/frontend/src/components/InceptionCard.vue +++ b/frontend/src/components/InceptionCard.vue @@ -28,7 +28,6 @@ const emit = defineEmits<{ }>(); const local = ref(props.choices ? { ...props.choices } : emptyChoices()); -const alwaysOn = ref<{ id: number; title: string }[]>([]); const others = ref<{ id: number; title: string }[]>([]); const designSystems = ref<{ id: number; title: string }[]>([]); const systemsCount = ref(0); @@ -47,21 +46,18 @@ async function load() { try { if (props.mode === "decide" && props.projectId) { const d: InceptionDefaults = await fetchInceptionDefaults(props.projectId); - alwaysOn.value = d.always_on_rulebooks; - others.value = d.other_rulebooks; + others.value = d.rulebooks; designSystems.value = d.design_systems; systemsCount.value = d.systems; // Start from what stands today so "record" without changes is a true inherit-all. local.value = { - exclude_always_on_rulebooks: d.excluded_always_on.map((r) => r.id), subscribe_rulebooks: d.subscribed_rulebooks.map((r) => r.id), design_system_id: d.design_system_id, seed_systems: false, }; } else { const [rulebooks, ds] = await Promise.all([listRulebooks(), fetchDesignSystems()]); - alwaysOn.value = rulebooks.filter((r) => r.always_on).map((r) => ({ id: r.id, title: r.title })); - others.value = rulebooks.filter((r) => !r.always_on).map((r) => ({ id: r.id, title: r.title })); + others.value = rulebooks.map((r) => ({ id: r.id, title: r.title })); designSystems.value = ds.design_systems.map((d) => ({ id: d.id, title: d.title })); } } catch (e: unknown) { @@ -71,13 +67,6 @@ async function load() { } } -function inherits(id: number): boolean { - return !local.value.exclude_always_on_rulebooks.includes(id); -} -function toggleInherit(id: number) { - const list = local.value.exclude_always_on_rulebooks; - local.value.exclude_always_on_rulebooks = list.includes(id) ? list.filter((x) => x !== id) : [...list, id]; -} function subscribed(id: number): boolean { return local.value.subscribe_rulebooks.includes(id); } @@ -87,7 +76,7 @@ function toggleSubscribe(id: number) { } const nothingToDecide = computed( - () => !alwaysOn.value.length && !others.value.length && !designSystems.value.length, + () => !others.value.length && !designSystems.value.length, ); async function record() { @@ -118,16 +107,11 @@ onMounted(load);

Loading…

{{ error }}