From e2a084f1fbf7a113109bece4056dc83dbe66a4e5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 20 Aug 2026 08:18:36 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat(ledger):=20coverage=20self-seeds=20?= =?UTF-8?q?=E2=80=94=20enter=5Fproject=20background=20refresh=20+=20refres?= =?UTF-8?q?h=5Fpattern=5Fcoverage=20tool=20+=20shape-accounting=20skill=20?= =?UTF-8?q?(#2802,=20milestone=20294)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UI Refresh button must not be the only seed path (operator directive, hit live: the P7 backfill stalled waiting for a click). Three parts: - enter_project fire-and-forgets refresh_if_stale on the project OWNER — absent or day-old readouts recompute in the background (same spawn the webhook path uses), the enter stays fast, forge-less owners exit quietly (rule #115 baseline), and an in-flight guard keeps concurrent enters from fetching the same tarball N times. - refresh_pattern_coverage(project_id): the synchronous agent-facing form — write-gated, owner-keyring resolution, and ValueError messages that name the fix (add a connection / bind_repo) instead of measuring nothing silently. - plugin 0.1.33 ships the shape-accounting skill: the five statuses, the seed/todo/judge loop, and the derive-first rule, triggered by the coverage line or any proved code-to-canon relationship. Co-Authored-By: Claude Fable 5 --- plugin/.claude-plugin/plugin.json | 2 +- plugin/skills/shape-accounting/SKILL.md | 51 +++++++++++++++ src/scribe/mcp/tools/projects.py | 15 ++++- src/scribe/mcp/tools/shapes.py | 31 ++++++++- src/scribe/services/coverage.py | 86 ++++++++++++++++++++++++- tests/test_mcp_tool_projects.py | 34 ++++++++++ tests/test_pattern_coverage.py | 46 +++++++++++++ tests/test_shape_ledger.py | 2 +- 8 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 plugin/skills/shape-accounting/SKILL.md diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index fb5dc06..f699407 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "scribe", "description": "Scribe system-of-record for Claude Code: MCP tools over your notes/tasks/projects/rules, a session-start push channel that surfaces your always-on rules + active-project context, process-skills (writing-plans, systematic-debugging, verification, brainstorming, reusing-code), and your saved Scribe Processes auto-surfaced as skills (/scribe:sync). Replaces superpowers + file-memory with one app-backed plugin.", - "version": "0.1.32", + "version": "0.1.33", "author": { "name": "Bryan Van Deusen" }, "mcpServers": { "scribe": { diff --git a/plugin/skills/shape-accounting/SKILL.md b/plugin/skills/shape-accounting/SKILL.md new file mode 100644 index 0000000..9c4b2e5 --- /dev/null +++ b/plugin/skills/shape-accounting/SKILL.md @@ -0,0 +1,51 @@ +--- +name: shape-accounting +description: Use when a project's shape accounting needs attention — the pattern_coverage line from enter_project shows unclassified shapes or is missing on a forge-served project, the operator asks about coverage/accounting/canon, or you just proved a code-to-canon relationship (an audit enumerated call sites, a consolidation repointed consumers, a verify pass confirmed a helper's users). Triggers on "coverage", "accounted", "unclassified", "classify shapes", "what uses this", or finishing any consolidation. +--- + +# Shape accounting — every shape classified against canon + +The snippet library records **canon** (small); the shape ledger accounts for +**every extracted definition** in a project's bound repos (total). Each ledger +row carries a status: + +- `canonical` — IS a snippet's reference (the coverage sync stamps these + mechanically; you rarely set it). +- `instance` of snippet N — conforms to recorded canon. Canon in another + project counts (a family-level button shape fully accounts for a local use). +- `variant` of snippet N — a deliberate, named departure. **Reason required** + — the why IS the record. +- `exempt` — judged genuinely one-off. **Reason required.** A recorded + judgment, not silence — it stops the next pass re-litigating it. +- `unclassified` — nobody has judged it yet. **This is the todo list.** + +## The loop + +1. **Seed / refresh** — the ledger fills from coverage computation. Entering a + project triggers a background seed automatically; when you need it current + *now* (before a classification batch, or when the line is missing on a + forge-served project), call `refresh_pattern_coverage(project_id)` — it + returns the fresh accounting line. Takes seconds; it moves repo archives. +2. **Read the todo** — `list_shapes(project_id, status="unclassified")`, + optionally scoped by `path` to the directories the coverage line names as + largest. `snippet_id=N` reads a consumer map. +3. **Judge in batches** — `classify_shapes(project_id, [{path, symbol, + status, snippet_id?, reason?}])`. All-or-nothing: a bad item applies + nothing. Rows, never prose — a consumer list in a note or verification + detail cannot be sorted, queried, or diffed. + +## The derive-first rule + +N same-shaped occurrences matching **no** recorded canon is never N loose +classifications — it is a consolidation candidate: derive one reference from +the dominant form, `create_snippet` it, migrate the outliers, then classify +the rest as instances. Canon is determined from the code; consistency comes +from the derivation, not from asking permission. + +## What this buys + +Divergence becomes mechanical: when button B appears where button A is canon, +the ledger says *unintended divergence* or *justified variant with its +reason* — nobody re-derives the history. `get_snippet` shows each snippet's +`instances` and `variants`, so "what uses this?" is answered from rows before +any contract change lands on its consumers. diff --git a/src/scribe/mcp/tools/projects.py b/src/scribe/mcp/tools/projects.py index 5a5fc7b..490ca52 100644 --- a/src/scribe/mcp/tools/projects.py +++ b/src/scribe/mcp/tools/projects.py @@ -26,6 +26,7 @@ from scribe.services import projects as projects_svc from scribe.services import rulebooks as rulebooks_svc from scribe.services import systems as systems_svc from scribe.services import trash as trash_svc +from scribe.services.background import spawn from scribe.services.note_usage import record_surfaced @@ -68,7 +69,9 @@ async def enter_project(project_id: int) -> dict: unclassified, largest: internal/api". Unclassified IS the todo: as you touch code in those areas, classify the shapes you can (instances of recorded canon, deliberate variants, one-off exemptions) and record the - canon that's missing with create_snippet. + canon that's missing with create_snippet. A null line on a forge-served + project usually means the ledger is seeding in the background (entering + triggers it); refresh_pattern_coverage computes it on the spot. `systems` is the project's vocabulary of named subsystems/areas. It is returned here so you can TAG as you write: when creating or meaningfully @@ -164,6 +167,16 @@ async def enter_project(project_id: int) -> dict: coverage = await coverage_svc.cached_coverage( project.user_id or uid, project_id ) + # Arrival self-seed (#2802): a ledger that is absent or stale refreshes in + # the BACKGROUND — entering is the moment the number is wanted, and the + # UI button must not be the only path. This enter stays fast; the next + # one carries the line. Forge-less projects exit the seed quietly. + spawn( + coverage_svc.refresh_if_stale( + project.user_id or uid, project_id, cached=coverage + ), + site="enter_project.coverage_seed", + ) out = { "project": project.to_dict(), diff --git a/src/scribe/mcp/tools/shapes.py b/src/scribe/mcp/tools/shapes.py index 6643ed1..19ed37c 100644 --- a/src/scribe/mcp/tools/shapes.py +++ b/src/scribe/mcp/tools/shapes.py @@ -9,6 +9,7 @@ refresh; these tools only ever judge what the sync has seen. from __future__ import annotations from scribe.mcp._context import current_user_id +from scribe.services import coverage as coverage_svc from scribe.services import shape_ledger as shape_ledger_svc @@ -90,6 +91,34 @@ async def list_shapes( return {"shapes": [r.to_dict() for r in rows], "total": total} +async def refresh_pattern_coverage(project_id: int) -> dict: + """Seed or refresh the project's shape ledger NOW, and return the readout. + + The synchronous form of the arrival-moment background seed (#2802): + downloads the bound repos through the OWNER's forge connections, upserts + every extracted shape into the ledger (new shapes arrive `unclassified`), + re-stamps snippet reference locations as canonical, and recomputes the + accounting. Reach for it when the ledger must be current before you act — + a classification batch about to run, a coverage question asked directly — + rather than waiting on the background seed an enter_project triggers. + + Takes seconds, not milliseconds (it moves repo archives). Requires write + access to the project. Errors name the fix: no forge connection → the + owner adds one (Settings → Integrations → Git Forges); no served repo → + bind_repo on a host a connection serves. + + Returns the accounting payload — total, accounted, counts by status, + unclassified, repos, largest_gaps — plus `pattern_coverage`, the same + one-line summary enter_project carries. + """ + uid = current_user_id() + coverage = await coverage_svc.refresh_for_caller(uid, project_id) + return { + "pattern_coverage": coverage_svc.coverage_line(coverage), + **coverage, + } + + def register(mcp) -> None: - for fn in (classify_shapes, list_shapes): + for fn in (classify_shapes, list_shapes, refresh_pattern_coverage): mcp.tool(name=fn.__name__)(fn) diff --git a/src/scribe/services/coverage.py b/src/scribe/services/coverage.py index cf2b623..04b13c0 100644 --- a/src/scribe/services/coverage.py +++ b/src/scribe/services/coverage.py @@ -31,7 +31,7 @@ import logging import posixpath import re import tarfile -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from scribe.services.forge import ForgeSelector, get_forges from scribe.services.repo_bindings import keys_for_project @@ -323,6 +323,90 @@ async def refresh_coverage( return coverage +# The arrival-moment self-seed (#2802). A ledger that has never been computed +# used to wait for someone to find the UI Refresh button; entering the project +# is the moment the number is wanted, so entering is the moment it seeds. +_SEED_MAX_AGE = timedelta(hours=24) + +# Projects with a refresh already running — concurrent enters must not fetch +# the same tarball N times. In-process on purpose: the cost being bounded is +# per-process forge traffic, and a rare double-fetch across workers is +# harmless (the upsert is idempotent). +_inflight_seed: set[int] = set() + + +async def refresh_if_stale( + user_id: int, project_id: int, cached: dict | None = None +) -> None: + """Background-refresh a project's ledger when its readout is absent or + older than a day. Fire-and-forget material (background.spawn): every exit + is quiet, every failure a WARNING — a seed must never surface as an + enter_project error. ``user_id`` is the project OWNER (whose keyring and + cache this is); pass ``cached`` when the caller already read it. + """ + try: + if cached is None: + cached = await cached_coverage(user_id, project_id) + if cached: + try: + computed = datetime.fromisoformat(cached.get("computed_at") or "") + if datetime.now(timezone.utc) - computed < _SEED_MAX_AGE: + return + except ValueError: + pass # an unreadable stamp reads as stale + if project_id in _inflight_seed: + return + selector = await get_forges(user_id, project_id) + if not selector.configured: + return # rule #115: forge-less stays exactly as it was + _inflight_seed.add(project_id) + try: + await refresh_coverage(user_id, project_id, selector=selector) + finally: + _inflight_seed.discard(project_id) + except Exception: + logger.warning( + "background coverage seed failed for project %s", project_id, + exc_info=True, + ) + + +async def refresh_for_caller(caller_id: int, project_id: int) -> dict: + """The explicit agent-facing refresh (#2802) — synchronous, named errors. + + Write-gated: recomputing spends the owner's forge API budget and rewrites + ledger rows' seen-markers, so a read share doesn't grant it. Resolution + runs on the OWNER's keyring like every other forge read. Raises + ValueError with a fixable message instead of silently measuring nothing — + the caller is an agent mid-task, and "None" would strand it exactly the + way the button-only path did. + """ + from scribe.models import async_session + from scribe.models.project import Project + from scribe.services import access + + if not await access.can_write_project(caller_id, project_id): + raise ValueError(f"project {project_id} not found or no write access") + async with async_session() as session: + project = await session.get(Project, project_id) + if project is None: + raise ValueError(f"project {project_id} not found") + owner_id = project.user_id or caller_id + selector = await get_forges(owner_id, project_id) + if not selector.configured: + raise ValueError( + "No forge connection serves this project — the owner adds one " + "under Settings → Integrations → Git Forges" + ) + coverage = await refresh_coverage(owner_id, project_id, selector=selector) + if coverage is None: + raise ValueError( + "No bound repo is served by the owner's forge connections — " + "bind_repo the project's repo on a host a connection serves" + ) + return coverage + + async def cached_coverage(user_id: int, project_id: int) -> dict | None: """The last computed summary, or None — never computes.""" raw = await get_setting(user_id, f"{_CACHE_KEY_PREFIX}{project_id}", "") diff --git a/tests/test_mcp_tool_projects.py b/tests/test_mcp_tool_projects.py index b464370..28e9f70 100644 --- a/tests/test_mcp_tool_projects.py +++ b/tests/test_mcp_tool_projects.py @@ -40,6 +40,15 @@ def _no_coverage(): yield +@pytest.fixture(autouse=True) +def _no_background_seed(): + """enter_project now fire-and-forgets a coverage self-seed (#2802). These + are no-database unit tests, so the spawn is stubbed out; the firing shape + has its own test below.""" + with patch("scribe.mcp.tools.projects.spawn") as mock: + yield mock + + @pytest.fixture(autouse=True) def _no_bootstrap(): """With _no_systems stubbing an empty vocabulary, every test here reaches @@ -319,6 +328,31 @@ async def test_enter_project_never_asks_bootstrap_once_a_vocabulary_exists( )) out = await enter_project(project_id=5) assert "systems_bootstrap" not in out + + +@pytest.mark.asyncio +async def test_enter_project_fires_the_coverage_seed_on_the_owner( + _no_background_seed, +): + """The arrival self-seed (#2802): entering spawns refresh_if_stale on the + project OWNER's id with the cache read the enter already did — fire and + forget, so a missing ledger seeds itself without the UI button, and the + enter stays fast.""" + import contextlib + + project = _fake_project(id=5) + project.user_id = 42 # explicit: the OWNER, not the caller (ctx uid=7) + with contextlib.ExitStack() as stack: + for cm in _enter_project_stubs(project): + stack.enter_context(cm) + seed = stack.enter_context(patch( + "scribe.mcp.tools.projects.coverage_svc.refresh_if_stale", + MagicMock(return_value=object()), + )) + await enter_project(project_id=5) + seed.assert_called_once_with(42, 5, cached=None) + _no_background_seed.assert_called_once() + assert _no_background_seed.call_args.kwargs["site"] == "enter_project.coverage_seed" _no_bootstrap.assert_not_awaited() diff --git a/tests/test_pattern_coverage.py b/tests/test_pattern_coverage.py index 1f08730..f5822a1 100644 --- a/tests/test_pattern_coverage.py +++ b/tests/test_pattern_coverage.py @@ -390,6 +390,52 @@ async def test_enter_project_surfaces_the_line_only_once_computed(seeded): _user_id_ctx.reset(token) +@pytest.mark.integration +async def test_explicit_refresh_names_its_failures(seeded): + """refresh_for_caller (#2802) raises fixable errors instead of silence: + an agent mid-task must learn WHY nothing measured — 'None' is exactly the + stranding the button-only path caused.""" + from scribe.models import async_session + from scribe.models.user import User + from scribe.services.coverage import refresh_for_caller + from sqlalchemy import select + + uid, pid = seeded["uid"], seeded["pid"] + # The owner has no forge connection rows → the error names the fix. + with pytest.raises(ValueError) as err: + await refresh_for_caller(uid, pid) + assert "Git Forges" in str(err.value) + + # A stranger gets not-found/no-write, never a measurement. + async with async_session() as s: + other = (await s.execute( + select(User).where(User.username == "coverage_outsider") + )).scalar_one_or_none() + if other is None: + other = User(username="coverage_outsider") + s.add(other) + await s.flush() + other_id = other.id + await s.commit() + with pytest.raises(ValueError) as err: + await refresh_for_caller(other_id, pid) + assert "no write access" in str(err.value) + + +@pytest.mark.integration +async def test_background_seed_is_quiet_without_a_forge(seeded): + """refresh_if_stale (#2802) must exit silently for a forge-less owner — + rule #115's baseline — and treat a fresh cache as nothing-to-do.""" + from scribe.services.coverage import refresh_coverage, refresh_if_stale + + uid, pid = seeded["uid"], seeded["pid"] + # Absent cache + no forge rows: returns without raising, writes nothing. + await refresh_if_stale(uid, pid) + # Fresh cache: returns before ever consulting the keyring. + stored = await refresh_coverage(uid, pid, selector=_selector(_tarball(TREE))) + await refresh_if_stale(uid, pid, cached=stored) + + @pytest.mark.integration async def test_unservable_binding_measures_nothing(seeded): """A project bound only to a host the forge doesn't serve returns None — diff --git a/tests/test_shape_ledger.py b/tests/test_shape_ledger.py index 56b0134..e3b95b2 100644 --- a/tests/test_shape_ledger.py +++ b/tests/test_shape_ledger.py @@ -80,5 +80,5 @@ def test_classify_and_list_are_mounted_as_mcp_tools(): from scribe.mcp.server import build_mcp_server mcp = build_mcp_server() - for name in ("classify_shapes", "list_shapes"): + for name in ("classify_shapes", "list_shapes", "refresh_pattern_coverage"): assert mcp._tool_manager.get_tool(name) is not None -- 2.54.0 From 3e21978a9c2f4ac51c574e72ba358ca698f3488a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 20 Aug 2026 08:24:05 -0400 Subject: [PATCH 2/2] fix(tests): restore the assert my seed-test insertion orphaned from its neighbor Co-Authored-By: Claude Fable 5 --- tests/test_mcp_tool_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mcp_tool_projects.py b/tests/test_mcp_tool_projects.py index 28e9f70..8f43cd8 100644 --- a/tests/test_mcp_tool_projects.py +++ b/tests/test_mcp_tool_projects.py @@ -328,6 +328,7 @@ async def test_enter_project_never_asks_bootstrap_once_a_vocabulary_exists( )) out = await enter_project(project_id=5) assert "systems_bootstrap" not in out + _no_bootstrap.assert_not_awaited() @pytest.mark.asyncio @@ -353,7 +354,6 @@ async def test_enter_project_fires_the_coverage_seed_on_the_owner( seed.assert_called_once_with(42, 5, cached=None) _no_background_seed.assert_called_once() assert _no_background_seed.call_args.kwargs["site"] == "enter_project.coverage_seed" - _no_bootstrap.assert_not_awaited() @pytest.mark.asyncio -- 2.54.0