feat(design-systems): check the components against the sheet they claim to use
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 8s
CI & Build / integration (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 43s
CI & Build / Build & push image (push) Successful in 35s

"The snippets use the tags from the sheet" was a relation nobody could verify.
Now it is three checks, and all three currently fail SILENTLY in this codebase:

  unknown             `var(--x)` where the system declares no `--x`. Renders as
                      nothing at all — no error, no failing test, no visual clue
                      beyond the element quietly not being styled.
  superseded literals a value the sheet said to stop writing, paired with the
                      token to write instead. Only possible because `supersedes`
                      is declared rather than inferred.
  local definitions   custom properties a snippet mints for itself instead of
                      reusing the sheet's — the bloat a shared sheet exists to
                      prevent, where a value stops being reused and starts being
                      restated per component.

The first is not hypothetical. Writing DesignSystemsView.vue earlier in this
same session I used `--color-accent` throughout; it does not exist, and nothing
in the toolchain noticed. This check is the thing that would have.

A token that is both defined and read locally is reported ONCE, as an unknown
reference — "--btn-bg does not exist in the sheet" is the more precise statement
of the same problem, and reporting both would double-count one fact.

Literal matching is boundary-aware and case-insensitive: `#fff` must not fire
inside `#ffffff` (different colours, and a finding on the wrong one sends
someone to change correct code), while `#FFFFFF` in a rulebook has to match
`#ffffff` in a stylesheet — the same trap `normalize_hex` exists for.

Snippets with nothing to report are omitted entirely. A list of everything that
is fine is a list nobody reads twice — the same principle the auto-inject menu
and the drift panel are both built on.

Two integration mistakes fixed while wiring it: `list_snippets` returns
`(rows, total)` and caps its limit at 100, and `get_snippet` returns a Note
model rather than a dict. The list rows carry a preview, not the code, so the
check reads each full body — checking the preview would have reported on a
truncation.
This commit is contained in:
2026-07-30 21:47:47 -04:00
parent b0a7d9e89b
commit 46d88f9e7e
8 changed files with 448 additions and 2 deletions
+24
View File
@@ -178,3 +178,27 @@ export interface StylesheetResult {
* rather than restated per element. */ * rather than restated per element. */
export const fetchStylesheet = (id: number) => export const fetchStylesheet = (id: number) =>
apiGet<StylesheetResult>(`/api/design-systems/${id}/stylesheet`); apiGet<StylesheetResult>(`/api/design-systems/${id}/stylesheet`);
export interface SnippetFinding {
snippet_id: number;
title: string;
/** References that resolve against the sheet. */
used: string[];
/** `var(--x)` where the system has no `--x` — renders as nothing at all. */
unknown: string[];
/** Literals the sheet says to stop writing, paired with what to write. */
superseded_literals: { literal: string; use_instead: string }[];
/** Custom properties the snippet mints for itself instead of reusing. */
local_definitions: string[];
}
export interface SnippetCheck {
design_system_id: number;
checked: number;
/** Only snippets with something to act on; clean ones are omitted. */
findings: SnippetFinding[];
}
/** Which recorded snippets disagree with this design system's sheet. */
export const checkSnippets = (id: number) =>
apiGet<SnippetCheck>(`/api/design-systems/${id}/snippet-check`);
+134
View File
@@ -29,6 +29,7 @@ import {
fetchDesignSystems, fetchDesignSystems,
fetchDesignTokens, fetchDesignTokens,
fetchResolvedTokens, fetchResolvedTokens,
checkSnippets,
fetchStylesheet, fetchStylesheet,
importFromRulebook, importFromRulebook,
updateDesignSystem, updateDesignSystem,
@@ -37,6 +38,7 @@ import {
type DesignToken, type DesignToken,
type ImportReport, type ImportReport,
type ResolvedToken, type ResolvedToken,
type SnippetCheck,
type StylesheetResult, type StylesheetResult,
} from "@/api/designSystems"; } from "@/api/designSystems";
import { listRulebooks, type Rulebook } from "@/api/rulebooks"; import { listRulebooks, type Rulebook } from "@/api/rulebooks";
@@ -433,6 +435,33 @@ watch([selectedId, ownTokens], () => {
const duplicateEntries = computed(() => Object.entries(sheet.value?.duplicates ?? {})); const duplicateEntries = computed(() => Object.entries(sheet.value?.duplicates ?? {}));
// --- do the snippets use the sheet? -----------------------------------------
const snippetCheck = ref<SnippetCheck | null>(null);
const checkingSnippets = ref(false);
/** The component layer checked against the sheet.
*
* Run on demand rather than on open: it reads every snippet's full body, which
* is a real cost, and the answer only changes when the sheet or the snippets
* do. */
async function runSnippetCheck() {
if (selectedId.value === null || checkingSnippets.value) return;
checkingSnippets.value = true;
try {
snippetCheck.value = await checkSnippets(selectedId.value);
} catch {
snippetCheck.value = null;
toast.show("Couldn't check the snippets", "error");
} finally {
checkingSnippets.value = false;
}
}
watch(selectedId, () => {
snippetCheck.value = null;
});
// --- import from a rulebook ------------------------------------------------- // --- import from a rulebook -------------------------------------------------
const rulebooks = ref<Rulebook[]>([]); const rulebooks = ref<Rulebook[]>([]);
@@ -693,6 +722,66 @@ function isColourish(value: string): boolean {
</template> </template>
</section> </section>
<!-- Do the components use the sheet? -->
<section class="ds-section">
<div class="section-head">
<h2>Components using this sheet</h2>
<button
class="btn-ghost btn-small" :disabled="checkingSnippets"
@click="runSnippetCheck"
>
{{ checkingSnippets ? "Checking…" : "Check snippets" }}
</button>
</div>
<p class="section-note">
Snippets are the component layer — buttons, tables, input schemes —
and they are meant to use the tags this sheet declares. This finds
the three ways that goes wrong silently.
</p>
<template v-if="snippetCheck">
<p class="section-note">
<strong>{{ snippetCheck.checked }}</strong> snippets checked ·
<strong>{{ snippetCheck.findings.length }}</strong> with something
to act on
</p>
<p v-if="!snippetCheck.findings.length" class="muted">
Nothing to report. Every snippet checked uses tags this system
declares, writes no superseded literals, and mints no tokens of
its own.
</p>
<ul v-else class="finding-list">
<li v-for="f in snippetCheck.findings" :key="f.snippet_id">
<router-link :to="`/snippets/${f.snippet_id}`" class="finding-title">
{{ f.title || `Snippet #${f.snippet_id}` }}
</router-link>
<p v-if="f.unknown.length" class="finding-line">
<span class="spec-status violated">renders as nothing</span>
references
<code v-for="name in f.unknown" :key="name">{{ name }}</code>
— no such token in this system.
</p>
<p v-if="f.superseded_literals.length" class="finding-line">
<span class="spec-status missing">write the token instead</span>
<span v-for="s in f.superseded_literals" :key="s.literal">
<code>{{ s.literal }}</code> → <code>{{ s.use_instead }}</code>
</span>
</p>
<p v-if="f.local_definitions.length" class="finding-line">
<span class="spec-status missing">defines its own</span>
<code v-for="name in f.local_definitions" :key="name">{{ name }}</code>
— a value restated here rather than reused from the sheet.
</p>
</li>
</ul>
</template>
</section>
<!-- Import from a rulebook --> <!-- Import from a rulebook -->
<section class="ds-section"> <section class="ds-section">
<div class="section-head"> <div class="section-head">
@@ -1330,6 +1419,51 @@ function isColourish(value: string): boolean {
gap: 0.35rem; gap: 0.35rem;
} }
.finding-list {
list-style: none;
margin: 0;
padding: 0;
}
.finding-list li {
padding: 0.6rem 0;
border-bottom: 1px solid var(--color-border);
}
.finding-title {
font-weight: 500;
color: var(--color-text);
}
.finding-line {
margin: 0.3rem 0 0;
font-size: 0.85rem;
color: var(--color-text-secondary);
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.35rem;
}
.spec-status {
font-size: 0.7rem;
text-transform: uppercase;
letter-spacing: 0.06em;
padding: 0.1rem 0.45rem;
border-radius: var(--radius-sm);
flex: none;
}
.spec-status.violated {
background: var(--color-priority-high-bg);
color: var(--color-priority-high);
}
.spec-status.missing {
background: var(--color-priority-medium-bg);
color: var(--color-priority-medium);
}
.sheet { .sheet {
background: var(--color-bg); background: var(--color-bg);
border: 1px solid var(--color-border); border: 1px solid var(--color-border);
+36
View File
@@ -167,6 +167,41 @@ async def get_design_system_stylesheet(
return result return result
async def check_snippets_against_design_system(
design_system_id: int,
project_id: int = 0,
) -> dict:
"""Which recorded snippets disagree with a design system's sheet.
Snippets are the component layer — buttons, tables, input schemes — and they
are supposed to use the tags the sheet declares. Three findings per snippet,
each currently silent in the codebase:
unknown `var(--x)` where the system has no `--x`. Renders as
NOTHING: no error, no failing test, just an element
that quietly isn't styled.
superseded_literals a literal the sheet says to stop writing, paired with
the token to write instead.
local_definitions custom properties the snippet mints for itself rather
than using shared ones — the bloat a shared sheet
exists to prevent.
Snippets with nothing to report are omitted. Reach for this before writing
or reviewing component CSS.
Args:
design_system_id: The system whose sheet is authoritative.
project_id: Narrow to one project, or 0 for every project.
"""
uid = current_user_id()
result = await ds_svc.check_snippets_against_system(
uid, design_system_id, project_id
)
if result is None:
raise ValueError(f"design system {design_system_id} not found")
return result
async def import_design_system_from_rulebook( async def import_design_system_from_rulebook(
design_system_id: int, design_system_id: int,
rulebook_id: int, rulebook_id: int,
@@ -336,6 +371,7 @@ def register(mcp) -> None:
update_design_system, update_design_system,
delete_design_system, delete_design_system,
get_design_system_stylesheet, get_design_system_stylesheet,
check_snippets_against_design_system,
import_design_system_from_rulebook, import_design_system_from_rulebook,
create_design_token, create_design_token,
list_design_tokens, list_design_tokens,
+18
View File
@@ -135,6 +135,24 @@ async def get_design_system_stylesheet(design_system_id: int):
return jsonify(result) return jsonify(result)
@design_systems_bp.get("/design-systems/<int:design_system_id>/snippet-check")
@login_required
async def check_snippets_against_system(design_system_id: int):
"""Which recorded snippets disagree with this system's sheet.
`?project_id=` narrows to one project; omit it to check every project, which
is usually right — a component recorded elsewhere still has to use the same
tags.
"""
project_id = request.args.get("project_id", type=int) or 0
result = await ds_svc.check_snippets_against_system(
_uid(), design_system_id, project_id
)
if result is None:
return _not_found()
return jsonify(result)
@design_systems_bp.post("/design-systems/<int:design_system_id>/import") @design_systems_bp.post("/design-systems/<int:design_system_id>/import")
@login_required @login_required
async def import_design_system(design_system_id: int): async def import_design_system(design_system_id: int):
+81
View File
@@ -211,3 +211,84 @@ def duplicate_values(tokens: Sequence) -> dict[str, list[str]]:
continue continue
by_value.setdefault(str(base).strip().lower(), []).append(name) by_value.setdefault(str(base).strip().lower(), []).append(name)
return {value: names for value, names in by_value.items() if len(names) > 1} return {value: names for value, names in by_value.items() if len(names) > 1}
# ---------------------------------------------------------------------------
# Reading a sheet from the other side: does this code use it correctly?
# ---------------------------------------------------------------------------
#
# "The snippets use the tags from the sheet" is a verifiable relation, and
# nothing checked it before. Three questions, each a different failure:
#
# var(--x) where no --x exists -> renders as NOTHING. No error, no test
# failure, no visual clue beyond the thing
# silently not being styled.
# a superseded literal in code -> the value the sheet said to stop writing,
# and the sheet knows what to write instead.
# --x: declared inside a snippet -> a component minting its own token is the
# bloat a shared sheet exists to prevent.
#
# The first is not hypothetical: `--color-accent` was used throughout a new view
# in this codebase and does not exist.
_VAR_REFERENCE = re.compile(r"var\(\s*(--[A-Za-z0-9_-]+)")
_LOCAL_DEFINITION = re.compile(r"(?<![\w-])(--[A-Za-z0-9_-]+)\s*:")
def referenced_tokens(code: str) -> set[str]:
"""Every custom property the code reads through `var()`."""
return set(_VAR_REFERENCE.findall(code or ""))
def defined_tokens(code: str) -> set[str]:
"""Every custom property the code declares itself.
Excludes names it also reads: `--x: var(--x, fallback)` is a redeclaration
of something the sheet owns, which the unknown-reference check already
covers more precisely.
"""
return set(_LOCAL_DEFINITION.findall(code or "")) - referenced_tokens(code or "")
def _literal_pattern(literal: str) -> re.Pattern:
"""Match a literal value without matching a longer one that contains it.
`#fff` must not match inside `#ffffff` — they are different colours, and a
finding that fired on the wrong one would send someone to change code that
was already correct.
"""
escaped = re.escape(literal)
lead = r"(?<![0-9A-Za-z_#-])"
trail = r"(?![0-9A-Za-z_-])" if literal.startswith("#") else r"(?![0-9A-Za-z_-])"
return re.compile(lead + escaped + trail, re.IGNORECASE)
def check_code_against_tokens(code: str, tokens) -> dict:
"""What this code gets wrong about that token set.
`tokens` is any sequence with `.name`, `.value_by_mode` and `.supersedes` —
resolved tokens, or stored ones.
Reports rather than scores. Every finding here has a legitimate exception:
a snippet may target a system it isn't being checked against, and a literal
may be deliberate in a context the token doesn't cover. What it removes is
the SILENCE — all three currently fail with no signal at all.
"""
known = {getattr(t, "name", "") for t in tokens}
referenced = referenced_tokens(code)
superseded: list[dict] = []
for token in tokens:
for literal in getattr(token, "supersedes", None) or ():
if _literal_pattern(str(literal)).search(code or ""):
superseded.append({
"literal": literal,
"use_instead": getattr(token, "name", ""),
})
return {
"used": sorted(referenced & known),
"unknown": sorted(referenced - known),
"superseded_literals": superseded,
"local_definitions": sorted(defined_tokens(code)),
}
+59 -1
View File
@@ -21,7 +21,11 @@ from scribe.models import async_session
from scribe.models.design_system import DesignSystem, DesignToken from scribe.models.design_system import DesignSystem, DesignToken
from scribe.models.project import Project from scribe.models.project import Project
from scribe.services import access from scribe.services import access
from scribe.services.design_stylesheet import duplicate_values, render_stylesheet from scribe.services.design_stylesheet import (
check_code_against_tokens,
duplicate_values,
render_stylesheet,
)
from scribe.services.design_cascade import ( from scribe.services.design_cascade import (
ResolvedToken, ResolvedToken,
ancestry, ancestry,
@@ -437,3 +441,57 @@ async def stylesheet_for_system(
"valueless": [t.name for t in resolved if not t.value_by_mode], "valueless": [t.name for t in resolved if not t.value_by_mode],
"duplicates": duplicate_values(resolved), "duplicates": duplicate_values(resolved),
} }
async def check_snippets_against_system(
user_id: int, design_system_id: int, project_id: int = 0
) -> dict | None:
"""Which recorded snippets disagree with this design system's sheet.
The relation the operator named — "the snippets use the tags from the sheet"
— turned into a check. For each snippet: `var(--x)` references with no such
token, literals the sheet says to stop writing, and custom properties the
snippet mints for itself instead of using shared ones.
Snippets with nothing to report are omitted entirely. A list of everything
that is fine is a list nobody reads twice.
"""
resolved = await resolve_design_system(user_id, design_system_id)
if resolved is None:
return None
from scribe.services import snippets as snippets_svc
# `list_snippets` returns (rows, total) and caps limit at 100; `project_id`
# must be None — not 0 — to reach across every project, since 0 would filter
# to a project with that id.
rows, _total = await snippets_svc.list_snippets(
user_id=user_id, project_id=project_id or None, limit=100,
)
findings: list[dict] = []
for row in rows:
snippet_id = row.get("id")
if snippet_id is None:
continue
# The list rows carry a preview, not the code. The check has to read the
# whole body or it would report on a truncation.
note = await snippets_svc.get_snippet(user_id=user_id, snippet_id=int(snippet_id))
if note is None:
continue
report = check_code_against_tokens(note.body or "", resolved)
if not (
report["unknown"] or report["superseded_literals"] or report["local_definitions"]
):
continue
findings.append({
"snippet_id": int(snippet_id),
"title": note.title or "",
**report,
})
return {
"design_system_id": design_system_id,
"checked": len(rows),
"findings": findings,
}
+91
View File
@@ -8,6 +8,7 @@ declaration.
from types import SimpleNamespace from types import SimpleNamespace
from scribe.services.design_stylesheet import ( from scribe.services.design_stylesheet import (
check_code_against_tokens,
duplicate_values, duplicate_values,
is_valid_token_name, is_valid_token_name,
render_stylesheet, render_stylesheet,
@@ -228,3 +229,93 @@ def test_valueless_tokens_never_count_as_duplicates_of_each_other():
"""Otherwise every unfilled token would collide with every other one and the """Otherwise every unfilled token would collide with every other one and the
report would be nothing but noise on a fresh import.""" report would be nothing but noise on a fresh import."""
assert duplicate_values([_token("--fs-a", {}), _token("--fs-b", {})]) == {} assert duplicate_values([_token("--fs-a", {}), _token("--fs-b", {})]) == {}
# --- reading the sheet from the other side ----------------------------------
#
# "The snippets use the tags from the sheet" made checkable. All three findings
# below are currently SILENT in this codebase — no error, no failing test.
def _tok(name, base=None, supersedes=()):
return SimpleNamespace(
name=name,
value_by_mode={"base": base} if base else {},
supersedes=list(supersedes),
group_name=None, purpose=None,
)
SHEET = [
_tok("--fs-obsidian", "#14171a"),
_tok("--fs-parchment", "#e8e4d8", supersedes=["#fff", "#ffffff"]),
]
def test_a_reference_to_a_token_that_does_not_exist_is_reported():
"""THE bug this exists for. `var(--color-accent)` where no such token exists
renders as nothing at all — invisible focus rings, unstyled elements, no
error and no failing test. It happened in this codebase this session."""
report = check_code_against_tokens("outline: 2px solid var(--color-accent);", SHEET)
assert report["unknown"] == ["--color-accent"]
assert report["used"] == []
def test_a_reference_that_resolves_is_reported_as_used_not_as_a_finding():
report = check_code_against_tokens("background: var(--fs-obsidian);", SHEET)
assert report["used"] == ["--fs-obsidian"]
assert report["unknown"] == []
def test_a_superseded_literal_is_found_and_names_its_replacement():
"""The reframe paying off end to end: the finding says what to WRITE, not
merely what is wrong. That is only possible because the token declared it."""
report = check_code_against_tokens("color: #fff;", SHEET)
assert report["superseded_literals"] == [
{"literal": "#fff", "use_instead": "--fs-parchment"}
]
def test_a_shorter_hex_does_not_match_inside_a_longer_one():
"""`#fff` must not fire on `#ffffff` — different colours, and a finding on
the wrong one sends someone to change code that was already correct."""
report = check_code_against_tokens("color: #ffffffee;", SHEET)
assert [f["literal"] for f in report["superseded_literals"]] == []
def test_the_literal_match_is_case_insensitive():
"""Rulebooks write `#FFFFFF` and code writes `#ffffff`. A case-sensitive
check would silently find nothing — the same trap normalize_hex exists for."""
report = check_code_against_tokens("color: #FFFFFF;", SHEET)
assert report["superseded_literals"] == [
{"literal": "#ffffff", "use_instead": "--fs-parchment"}
]
def test_a_token_defined_and_read_locally_is_reported_once_not_twice():
"""A snippet minting its own custom property is the bloat a shared sheet
exists to prevent — but when it also reads it back, that is ONE fact. The
unknown-reference check owns it, because "--btn-bg does not exist in the
sheet" is the more precise statement of the same problem."""
report = check_code_against_tokens(".btn { --btn-bg: #333; background: var(--btn-bg); }", SHEET)
assert report["unknown"] == ["--btn-bg"]
assert report["local_definitions"] == []
def test_a_local_definition_never_read_back_is_still_reported():
report = check_code_against_tokens(".btn { --btn-bg: #333; }", SHEET)
assert report["local_definitions"] == ["--btn-bg"]
def test_a_declaration_that_is_not_a_custom_property_is_not_mistaken_for_one():
report = check_code_against_tokens(".btn { color: red; background: blue; }", SHEET)
assert report["local_definitions"] == []
def test_clean_code_reports_nothing_to_act_on():
report = check_code_against_tokens(
".btn { background: var(--fs-obsidian); color: var(--fs-parchment); }", SHEET
)
assert report["unknown"] == []
assert report["superseded_literals"] == []
assert report["local_definitions"] == []
assert report["used"] == ["--fs-obsidian", "--fs-parchment"]
+5 -1
View File
@@ -25,6 +25,7 @@ def test_route_handlers_callable():
"list_design_systems", "create_design_system", "get_design_system", "list_design_systems", "create_design_system", "get_design_system",
"update_design_system", "delete_design_system", "resolve_design_system", "update_design_system", "delete_design_system", "resolve_design_system",
"import_design_system", "get_design_system_stylesheet", "import_design_system", "get_design_system_stylesheet",
"check_snippets_against_system",
"list_design_tokens", "create_design_token", "list_design_tokens", "create_design_token",
"update_design_token", "delete_design_token", "set_project_design_system", "update_design_token", "delete_design_token", "set_project_design_system",
): ):
@@ -47,6 +48,7 @@ def test_every_endpoint_is_reachable_on_the_app():
"/api/design-systems/<int:design_system_id>/resolved", "/api/design-systems/<int:design_system_id>/resolved",
"/api/design-systems/<int:design_system_id>/import", "/api/design-systems/<int:design_system_id>/import",
"/api/design-systems/<int:design_system_id>/stylesheet", "/api/design-systems/<int:design_system_id>/stylesheet",
"/api/design-systems/<int:design_system_id>/snippet-check",
"/api/design-systems/<int:design_system_id>/tokens", "/api/design-systems/<int:design_system_id>/tokens",
"/api/design-tokens/<int:token_id>", "/api/design-tokens/<int:token_id>",
"/api/projects/<int:project_id>/design-system", "/api/projects/<int:project_id>/design-system",
@@ -61,7 +63,7 @@ def test_service_functions_take_user_id():
"update_design_system", "delete_design_system", "resolve_design_system", "update_design_system", "delete_design_system", "resolve_design_system",
"create_token", "list_tokens", "update_token", "delete_token", "create_token", "list_tokens", "update_token", "delete_token",
"set_project_design_system", "import_from_rulebook", "set_project_design_system", "import_from_rulebook",
"stylesheet_for_system", "stylesheet_for_system", "check_snippets_against_system",
): ):
fn = getattr(svc, fn_name) fn = getattr(svc, fn_name)
assert callable(fn) assert callable(fn)
@@ -94,6 +96,8 @@ def test_agent_and_web_surfaces_stay_at_parity():
# so the loop above can't pair it. It still has to exist on both. # so the loop above can't pair it. It still has to exist on both.
assert callable(tools.import_design_system_from_rulebook) assert callable(tools.import_design_system_from_rulebook)
assert callable(routes.import_design_system) assert callable(routes.import_design_system)
assert callable(tools.check_snippets_against_design_system)
assert callable(routes.check_snippets_against_system)
def test_every_mcp_tool_in_the_module_is_registered(): def test_every_mcp_tool_in_the_module_is_registered():