bvandeusenandClaude Opus 5.5
582a5a4f48
CI & Build / Python lint (push) Successful in 2s
CI & Build / Plugin hooks (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / integration (push) Successful in 47s
CI & Build / Python tests (push) Successful in 1m39s
CI & Build / Build & push image (push) Successful in 29s
- reusing-code: "Before the turn ends — say what you built" — the four verdicts, the one classify_shapes(repo=…) call, and the component file as a shape. Description names the end-of-turn moment. - shape-accounting: the writer judges; audits are the check that it held. The write path SUGGESTS (no more hook instances); component file rows and whole-file canon described; scoped covers Svelte too. - _INSTRUCTIONS reuse line: "before the turn ends, say what you built (create_snippet the reusable, classify_shapes the rest)" — 1570/1600. - Coverage line: "written-shape check (7d): N turns checked, M asked, K left unjudged", from the Stop hook's recorded outcomes; silent until the question has been put. - test_guidance_ownership pins the new topic on reusing-code. - prior-art hook header no longer says it stamps instance rows. Plugin version minted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1180 lines
52 KiB
Python
1180 lines
52 KiB
Python
"""Pattern-library coverage — what fraction of a bound repo's shapes have a
|
||
recorded snippet (#2692, forge job 3 of decision #2686).
|
||
|
||
The all-shapes doctrine says every shape gets recorded at first build. This
|
||
module is the hoping→knowing move: it enumerates the definitions that exist in
|
||
a project's bound repos (via the forge, one archive download per repo) and
|
||
compares them against recorded snippet locations, so "record everything"
|
||
becomes a watched number instead of an aspiration.
|
||
|
||
The definition extractor MIRRORS the write-path hook's awk rules
|
||
(plugin/hooks/scribe_prior_art.sh, ARM 1) — one shared notion of "a
|
||
definition" between the hook and the server, so the metric and the backstop
|
||
agree on what counts. The two are pinned together by shared test vectors in
|
||
tests/test_pattern_coverage.py; change one, change both.
|
||
|
||
The number is an ESTIMATE and every surface must say so: keyword extraction
|
||
over-counts (private one-offs, generated code that slips the dir filter) and
|
||
under-counts (keyword-less declaration syntax — C/Java/Dart — needs a real
|
||
parser and is out of scope, exactly as it is for the hook). The trend carries
|
||
the meaning, like the usage counters; the raw number is not a grade.
|
||
|
||
Compute is on demand + cached with a freshness stamp — recomputed on webhook
|
||
push and explicit refresh, NEVER in the request path of enter_project, which
|
||
only ever reads the cache.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import logging
|
||
import posixpath
|
||
import re
|
||
import tarfile
|
||
from typing import NamedTuple
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
from scribe.services.forge import ForgeSelector, get_forges
|
||
from scribe.services.repo_bindings import bindings_for_project
|
||
from scribe.services.settings import get_setting, set_setting
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Cache key in the settings KV, on the project OWNER's user_id — the same
|
||
# channel the scheduler's last-run summary uses for machine-written state.
|
||
# v2 suffix with #2788: the payload shape inverted (accounted/unclassified);
|
||
# pre-ledger blobs under the old key simply stop being found, so the card
|
||
# honestly reads "not measured yet" until the first ledger-era refresh.
|
||
_CACHE_KEY_PREFIX = "pattern_coverage_v2_"
|
||
|
||
# Files whose content can't hold definitions — the hook's skip list, verbatim,
|
||
# plus sourcemaps (which are JSON in a trenchcoat).
|
||
_SKIP_SUFFIXES = (
|
||
".md", ".mdx", ".txt", ".rst", ".json", ".lock", ".log", ".csv", ".tsv",
|
||
".svg", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".map",
|
||
)
|
||
|
||
# Vendored/generated trees would swamp the metric with shapes nobody should
|
||
# record — the dunder-skip lesson at directory scale: guaranteed noise teaches
|
||
# people to ignore the number.
|
||
_SKIP_DIRS = frozenset({
|
||
"node_modules", "vendor", "dist", "build", "target",
|
||
"__pycache__", ".git", ".venv", "venv",
|
||
})
|
||
|
||
# A single source file bigger than this is almost certainly generated or
|
||
# vendored (bundles, lockstep protos) — skipped, and part of why the number
|
||
# is labeled an estimate.
|
||
_MAX_FILE_BYTES = 1_000_000
|
||
|
||
|
||
# --- the definition extractor (mirror of scribe_prior_art.sh ARM 1) ----------
|
||
|
||
_CSS_RE = re.compile(r"^\s*\.([A-Za-z][A-Za-z0-9_-]*)\s*[,{]")
|
||
# Leading declaration modifiers, so the definition keyword is the first word
|
||
# regardless of language (export/pub/private/suspend/...).
|
||
_MODIFIERS_RE = re.compile(
|
||
r"^(?:(?:pub(?:\([a-z]+\))?|export|default|private|internal|protected"
|
||
r"|public|static|suspend|async|open|sealed|data|abstract|final|inline"
|
||
r"|unsafe|extern|override)\s+)*"
|
||
)
|
||
# Go method with receiver: func (r *T) Name(
|
||
_GO_METHOD_RE = re.compile(r"^func\s*\([^)]*\)\s*([A-Za-z_][A-Za-z0-9_]*)")
|
||
# Keyword-announced definitions, functions and named types alike. `impl` is
|
||
# excluded on purpose — several per type is normal Rust, not duplication.
|
||
_KEYWORD_RE = re.compile(
|
||
r"^(?:function|def|class|func|fun|fn|sub|struct|trait|interface|enum"
|
||
r"|object|protocol|type)\s+([A-Za-z_$][A-Za-z0-9_$]*)"
|
||
)
|
||
# Arrow/expression assignment: const name = (…) / let name = async (
|
||
_ARROW_RE = re.compile(
|
||
r"^(?:const|let)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?[(<]"
|
||
)
|
||
|
||
|
||
# --- comment and string spans (#4222) ----------------------------------------
|
||
#
|
||
# The matchers above are line-oriented and know nothing about what a line is
|
||
# INSIDE. A wrapped docstring whose line happens to begin "class AND the …"
|
||
# reads as a definition of a shape called `AND`, and that phantom reaches the
|
||
# agent mid-edit as a divergence prompt about a symbol that does not exist.
|
||
# This repo's own source minted twenty-two of them, measured by running the
|
||
# extractor over every scannable file with and without this scan. Two —
|
||
# `with` and `nobody`, out of one module docstring in
|
||
# scripts/check_dangling_styles.py — are persisted, JUDGED `code_shapes`
|
||
# rows, so the cost was never only noise in the moment. Those clear
|
||
# themselves: sync_shapes marks a row it no longer extracts as vanished.
|
||
#
|
||
# The honest tool for .py would be `ast`, which cannot be fooled by prose at
|
||
# all. It is not what this uses, because this extractor is mirrored rule for
|
||
# rule by an awk program in plugin/hooks/scribe_defs.sh, awk cannot parse
|
||
# Python, and a fix only one of the pair can run is the drift the mirror
|
||
# exists to prevent. What both can do is blank the SPANS: a triple-quoted
|
||
# string or a /* … */ comment is replaced by its own newlines before any
|
||
# matcher sees a line, so every line index still lines up and the signature,
|
||
# body and fingerprint keep reading the untouched original.
|
||
#
|
||
# ONLY CLOSED SPANS ARE BLANKED, and an unterminated opener is stepped over
|
||
# rather than bailed on, so a stray opener costs one mishandled span and never
|
||
# every definition below it in the file.
|
||
_SPANS = (('"""', '"""'), ("'''", "'''"), ("/*", "*/"))
|
||
|
||
|
||
def _string_end(text: str, at: int) -> int:
|
||
"""Offset just past the one-line string opening at ``at``.
|
||
|
||
``at`` + 1 when it does not close before the newline, so an apostrophe in
|
||
prose — `don't` in a Vue template, outside any comment — costs one
|
||
character rather than everything up to the next quote in the file.
|
||
"""
|
||
quote = text[at]
|
||
i = at + 1
|
||
while i < len(text):
|
||
c = text[i]
|
||
if c == "\\":
|
||
i += 2
|
||
elif c == "\n":
|
||
return at + 1
|
||
elif c == quote:
|
||
return i + 1
|
||
else:
|
||
i += 1
|
||
return at + 1
|
||
|
||
|
||
def _hash_comment(text: str, at: int) -> bool:
|
||
"""Does the `#` at ``at`` open a comment, or is it a CSS colour or id?
|
||
|
||
`#` is the one marker whose meaning depends on the language, and the
|
||
extractor is handed text with no path. An alphanumeric straight after it
|
||
is `#fff` or `#app`; anything else — a space, a `!`, a `-` — is a comment
|
||
in every language that has one.
|
||
"""
|
||
nxt = text[at + 1:at + 2]
|
||
return not nxt.isalnum()
|
||
|
||
|
||
def _blank_spans(text: str) -> str:
|
||
"""``text`` with comment and string spans replaced by their own newlines.
|
||
|
||
A single left-to-right pass, because the alternative — matching markers
|
||
wherever they appear — cannot tell a comment from a string that QUOTES
|
||
one. Both of those are in this repo: the docstring-matching regex in
|
||
plugin_context.py holds a triple quote inside a single-quoted literal,
|
||
and test_design_stylesheet.py asserts on the text `red /*` inside a
|
||
double-quoted one. Each cost every definition below it in its file before
|
||
the scan was written this way.
|
||
|
||
Line COUNT is preserved, column positions are not — the result is only
|
||
ever fed to the line matchers, which lstrip anyway.
|
||
"""
|
||
out: list[str] = []
|
||
i = cut = 0
|
||
n = len(text)
|
||
while i < n:
|
||
opener = closer = ""
|
||
for op, cl in _SPANS:
|
||
if text.startswith(op, i):
|
||
opener, closer = op, cl
|
||
break
|
||
if opener:
|
||
end = text.find(closer, i + len(opener))
|
||
if end < 0:
|
||
# Unterminated: step over the opener rather than bail, so a
|
||
# stray marker costs one span and not the rest of the file.
|
||
i += len(opener)
|
||
continue
|
||
end += len(closer)
|
||
out.append(text[cut:i])
|
||
out.append("\n" * text.count("\n", i, end))
|
||
i = cut = end
|
||
continue
|
||
if text.startswith("//", i) or (text[i] == "#" and _hash_comment(text, i)):
|
||
# A line comment is COPIED, not blanked: its continuation lines
|
||
# carry their own marker, so none of them can read as a
|
||
# definition on their own.
|
||
nl = text.find("\n", i)
|
||
i = n if nl < 0 else nl
|
||
continue
|
||
if text[i] in "\"'":
|
||
i = _string_end(text, i)
|
||
continue
|
||
i += 1
|
||
out.append(text[cut:])
|
||
return "".join(out)
|
||
|
||
|
||
def _masked_lines(text: str, count: int) -> list[str]:
|
||
"""``text`` blanked and split, reconciled to ``count`` lines.
|
||
|
||
Blanking preserves every ``\n``, but ``splitlines`` also breaks on a bare
|
||
``\r`` and on the vertical-tab family, which a blanked span drops. The
|
||
reconciliation is what keeps a span containing one of those from shifting
|
||
suppression onto the wrong lines — padding is short by a line, never
|
||
misaligned by one.
|
||
"""
|
||
masked = _blank_spans(text).splitlines()
|
||
if len(masked) < count:
|
||
masked += [""] * (count - len(masked))
|
||
return masked[:count]
|
||
|
||
|
||
class Definition(NamedTuple):
|
||
"""One extracted definition with its content fingerprint (#2792).
|
||
|
||
`signature` is the definition line itself; `body_sha` hashes the block
|
||
whitespace- and comment-insensitively; `body` is the block's text, held
|
||
only for the duration of a refresh (the proposer matches on it) and
|
||
never stored.
|
||
"""
|
||
|
||
kind: str
|
||
name: str
|
||
signature: str
|
||
body_sha: str
|
||
body: str
|
||
line: int = -1 # 0-based line the definition starts on (#2869)
|
||
|
||
|
||
def _definition_on(raw: str) -> tuple[str, str] | None:
|
||
"""The (kind, name) this one line defines, or None. First match wins —
|
||
the same order the hook's awk program tries."""
|
||
m = _CSS_RE.match(raw)
|
||
if m:
|
||
return ("css", m.group(1))
|
||
line = _MODIFIERS_RE.sub("", raw.lstrip())
|
||
if m := _GO_METHOD_RE.match(line):
|
||
return ("sym", m.group(1))
|
||
if m := _KEYWORD_RE.match(line):
|
||
name = m.group(1)
|
||
if name.startswith("__") and name.endswith("__"):
|
||
return None
|
||
# `type` announces a definition only when something is declared after
|
||
# the name (`type Foo = …`, `type Foo struct {`); an import specifier
|
||
# (`import { type Foo, bar }`) is the same two words and defines
|
||
# nothing — it showed up as a two-file "identical body" family (#2904).
|
||
if line.startswith("type") and not re.search(r"[={]", line[m.end():]):
|
||
return None
|
||
return ("sym", name)
|
||
if m := _ARROW_RE.match(line):
|
||
return ("sym", m.group(1))
|
||
return None
|
||
|
||
|
||
# A definition's block runs from its line until the next non-blank line at
|
||
# its own indentation or shallower that is not a closer — so a Python def ends
|
||
# at the next top-level statement, a braces block keeps its `}`, a CSS rule
|
||
# keeps its `}`. Capped so a generated monolith can't make one shape's
|
||
# fingerprint cover the file.
|
||
_BLOCK_CAP = 120
|
||
_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);")
|
||
# Lines that don't change what a shape IS: comments and decorators. Dropped
|
||
# from the fingerprint so touching a comment above the next function doesn't
|
||
# read as this one's body changing.
|
||
_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "<!--", "-->")
|
||
_SIGNATURE_CAP = 300
|
||
|
||
|
||
def _indent(line: str) -> int:
|
||
return len(line) - len(line.lstrip())
|
||
|
||
|
||
def _block_sha(lines: list[str]) -> str:
|
||
kept = [
|
||
" ".join(ln.split())
|
||
for ln in lines
|
||
if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES)
|
||
]
|
||
return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16]
|
||
|
||
|
||
def _declaration_count(lines: list[str]) -> int:
|
||
"""How many `prop: value` declarations a CSS block body carries."""
|
||
body = " ".join(lines)
|
||
return sum(1 for part in body.replace("}", "").split(";") if ":" in part)
|
||
|
||
|
||
def extract_definitions(text: str) -> list[Definition]:
|
||
"""Every definition this text makes, with signature + fingerprint.
|
||
|
||
Duplicate (kind, name) within one text collapse to the first — the
|
||
ledger's identity is per file, so a second definition of the same name
|
||
(an overload, a re-declaration) is the same shape to it.
|
||
"""
|
||
lines = text.splitlines()
|
||
masked = _masked_lines(text, len(lines))
|
||
starts: list[tuple[int, str, str]] = []
|
||
for i, raw in enumerate(masked):
|
||
hit = _definition_on(raw)
|
||
if hit:
|
||
starts.append((i, hit[0], hit[1]))
|
||
seen: set[tuple[str, str]] = set()
|
||
out: list[Definition] = []
|
||
for i, kind, name in starts:
|
||
if (kind, name) in seen:
|
||
continue
|
||
seen.add((kind, name))
|
||
base = _indent(lines[i])
|
||
end = min(len(lines), i + _BLOCK_CAP)
|
||
for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)):
|
||
ln = lines[j]
|
||
if not ln.strip():
|
||
continue
|
||
if _indent(ln) <= base and ln.strip() not in _CLOSERS:
|
||
end = j
|
||
break
|
||
block = lines[i:end]
|
||
# A CSS rule's fingerprint is its DECLARATIONS, not its selector
|
||
# (#2872): the row's identity already carries the selector. Since
|
||
# note 2917 the derive grouping no longer reads CSS bodies at all (a
|
||
# class is grouped by name only), so for CSS the fingerprint is the
|
||
# recheck identity — "did this rule's body change since it was
|
||
# judged?" — and nothing more. The shape of the hash is kept as-is on
|
||
# purpose: changing it would flip every judged CSS row to recheck on
|
||
# the next sync. Sym blocks keep their signature line in the hash.
|
||
if kind == "css":
|
||
# One-line rules (`.x { color: red; }`) carry their declarations on
|
||
# the selector line itself; a block that is only the selector plus
|
||
# trailing blanks must not hash to the empty string (which grouped
|
||
# 68 unrelated one-liners as one "copy" on first deploy, #2872).
|
||
first = lines[i]
|
||
brace = first.find("{")
|
||
head = [first[brace + 1:]] if brace >= 0 and first[brace + 1:].strip() else []
|
||
hashed = head + block[1:]
|
||
if not any(x.strip() for x in hashed):
|
||
hashed = block
|
||
# A SINGLE declaration is not a shape (#2903): `color: var(--fs-
|
||
# text-tertiary)` under .text-muted, .task-mark and .pin-badge-auto
|
||
# is three meanings sharing one line, not three copies of one
|
||
# rule. Keep the selector in the hash for one-liners; two
|
||
# declarations and up stay selector-agnostic. (Moot for grouping
|
||
# since note 2917, kept for fingerprint stability — see above.)
|
||
elif _declaration_count(hashed) < 2:
|
||
hashed = block
|
||
else:
|
||
hashed = block
|
||
out.append(Definition(
|
||
kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(hashed),
|
||
"\n".join(block), i,
|
||
))
|
||
return out
|
||
|
||
|
||
# --- by-construction scope (#2869) -------------------------------------------
|
||
#
|
||
# A Vue single-file component's `<style scoped>` rules and its `<script setup>`
|
||
# functions cannot be reached from any other file: they are one-offs by
|
||
# construction, not by judgment. The sync stamps them `scoped` (mechanical) so
|
||
# the human todo holds only shapes a person should look at, while the bodies
|
||
# stay in play for the proposer, derive grouping and divergence — the five
|
||
# auth views' identical rules were found exactly there. Unscoped `<style>` in
|
||
# a .vue and every file that is neither .vue nor .svelte stay ordinary.
|
||
_STYLE_OPEN_RE = re.compile(r"^\s*<style\b[^>]*\bscoped\b", re.IGNORECASE)
|
||
_STYLE_CLOSE_RE = re.compile(r"^\s*</style\s*>", re.IGNORECASE)
|
||
|
||
# Svelte scopes the other way round (#4608): every `<style>` is component-
|
||
# scoped unless it opts out — `<style global>` (svelte-preprocess) for a whole
|
||
# block, `:global(...)` for one selector. Instance `<script>` code is the
|
||
# component's own; `<script context="module">` / `<script module>` is the one
|
||
# block other files can import from, so its definitions stay ordinary.
|
||
_SVELTE_STYLE_OPEN_RE = re.compile(r"^\s*<style\b(?![^>]*\bglobal\b)[^>]*>", re.IGNORECASE)
|
||
_SVELTE_SCRIPT_OPEN_RE = re.compile(r"^\s*<script\b([^>]*)>", re.IGNORECASE)
|
||
_SVELTE_MODULE_ATTR_RE = re.compile(r"""\bcontext\s*=\s*["']module["']|\bmodule\b""", re.IGNORECASE)
|
||
_SCRIPT_CLOSE_RE = re.compile(r"</script\s*>", re.IGNORECASE)
|
||
|
||
|
||
def _svelte_scoped(text: str, defs: list[Definition]) -> set[tuple[str, str]]:
|
||
"""`scoped_definitions` for a .svelte file: syms outside a module script,
|
||
css inside a non-global `<style>` that is not itself `:global(...)`."""
|
||
lines = text.splitlines()
|
||
styles: list[tuple[int, int]] = []
|
||
modules: list[tuple[int, int]] = []
|
||
style_at: int | None = None
|
||
module_at: int | None = None
|
||
for i, ln in enumerate(lines):
|
||
if style_at is None and _SVELTE_STYLE_OPEN_RE.match(ln):
|
||
style_at = i
|
||
elif style_at is not None and _STYLE_CLOSE_RE.match(ln):
|
||
styles.append((style_at, i))
|
||
style_at = None
|
||
m = _SVELTE_SCRIPT_OPEN_RE.match(ln)
|
||
if module_at is None and m and _SVELTE_MODULE_ATTR_RE.search(m.group(1)):
|
||
module_at = i
|
||
if module_at is not None and _SCRIPT_CLOSE_RE.search(ln):
|
||
modules.append((module_at, i))
|
||
module_at = None
|
||
out: set[tuple[str, str]] = set()
|
||
for d in defs:
|
||
if d.kind == "sym":
|
||
if not any(a <= d.line <= b for a, b in modules):
|
||
out.add((d.kind, d.name))
|
||
elif any(a <= d.line <= b for a, b in styles):
|
||
line = lines[d.line] if 0 <= d.line < len(lines) else ""
|
||
if ":global" not in line:
|
||
out.add((d.kind, d.name))
|
||
return out
|
||
|
||
|
||
def scoped_definitions(path: str, text: str, defs: list[Definition]) -> set[tuple[str, str]]:
|
||
"""The (kind, name) pairs among ``defs`` that are one-offs by
|
||
construction in this file: every sym in a .vue, and every css rule
|
||
that starts inside a `<style scoped>` block; for a .svelte, see
|
||
`_svelte_scoped`. Empty for other files."""
|
||
lowered = (path or "").lower()
|
||
if lowered.endswith(".svelte"):
|
||
return _svelte_scoped(text, defs)
|
||
if not lowered.endswith(".vue"):
|
||
return set()
|
||
ranges: list[tuple[int, int]] = []
|
||
open_at: int | None = None
|
||
for i, ln in enumerate(text.splitlines()):
|
||
if open_at is None and _STYLE_OPEN_RE.match(ln):
|
||
open_at = i
|
||
elif open_at is not None and _STYLE_CLOSE_RE.match(ln):
|
||
ranges.append((open_at, i))
|
||
open_at = None
|
||
out: set[tuple[str, str]] = set()
|
||
for d in defs:
|
||
if d.kind == "sym":
|
||
out.add((d.kind, d.name))
|
||
elif any(a <= d.line <= b for a, b in ranges):
|
||
out.add((d.kind, d.name))
|
||
return out
|
||
|
||
|
||
# --- template class references: the CSS consumer map (milestone 302) ---------
|
||
|
||
# Files whose MARKUP can consume a class. Styling consumers are templates —
|
||
# `querySelector('.x')` / classList in scripts are deliberately not read in
|
||
# v1 (note 2917: watch CSS by name, by recipe, by token and by what uses it;
|
||
# "what uses it" is the template).
|
||
_TEMPLATE_SUFFIXES = (
|
||
".vue", ".html", ".htm", ".jsx", ".tsx", ".js", ".ts", ".svelte", ".astro",
|
||
)
|
||
_CLASS_TOKEN_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
|
||
# Static: class="a b" / class='a b' / className="a b". The lookbehind keeps
|
||
# `:class=`, `v-bind:class=`, `data-class=` and `headerClass=` out of the
|
||
# static form (the Vue/React dynamic forms are read below; the others are
|
||
# not class attributes).
|
||
_STATIC_CLASS_RE = re.compile(
|
||
r"""(?<![:\w.-])(?:class|className)\s*=\s*(?:"([^"]*)"|'([^']*)')"""
|
||
)
|
||
# Dynamic: Vue `:class="…"` / `v-bind:class="…"`, React `className={…}` (one
|
||
# level of nested braces — an object literal inside the expression).
|
||
_DYNAMIC_CLASS_RE = re.compile(
|
||
r""":class\s*=\s*(?:"([^"]*)"|'([^']*)')"""
|
||
r"""|(?<![:\w.-])className\s*=\s*\{((?:[^{}]|\{[^{}]*\})*)\}"""
|
||
)
|
||
# Svelte's directive form: class:active={cond}.
|
||
_SVELTE_CLASS_RE = re.compile(r"(?<![:\w.-])class:([A-Za-z_][A-Za-z0-9_-]*)\s*=")
|
||
# Transition classes are applied by the FRAMEWORK, never written in markup:
|
||
# <Transition name="toast"> makes Vue add .toast-enter-active et al at
|
||
# runtime, and React's <CSSTransition classNames="fade"> does the same. A
|
||
# reader of `class=` attributes alone therefore calls every one of those
|
||
# rules unused, which is a false positive no amount of care in the
|
||
# stylesheet can avoid (#2970). A dynamic `:name="…"` stays unknowable.
|
||
_TRANSITION_NAME_RE = re.compile(
|
||
r"""<\s*[Tt]ransition(?:-[Gg]roup|Group)?\b[^>]*?(?<![:\w.-])name\s*=\s*"""
|
||
r"""(?:"([^"]*)"|'([^']*)')"""
|
||
r"""|(?<![:\w.-])classNames\s*=\s*(?:"([^"]*)"|'([^']*)')"""
|
||
)
|
||
# The union of what Vue 3, Vue 2 and React CSSTransition generate. Naming a
|
||
# class that no rule defines costs nothing — it resolves to no row — so the
|
||
# union is safer than guessing the framework from the file.
|
||
_TRANSITION_SUFFIXES = (
|
||
"-enter", "-enter-from", "-enter-active", "-enter-to", "-enter-done",
|
||
"-leave", "-leave-from", "-leave-active", "-leave-to",
|
||
"-exit", "-exit-active", "-exit-done",
|
||
"-appear", "-appear-from", "-appear-active", "-appear-to", "-appear-done",
|
||
"-move",
|
||
)
|
||
# A name built by concatenation — `status-${s}`, 'pri-' + p, class="c-{{ v }}"
|
||
# — leaves its static head behind once the hole is blanked. That head is a
|
||
# PREFIX reference, spelled `status-*`: "*" cannot occur in a class token, so
|
||
# the marker rides the plain token dict without a schema change. Needs a real
|
||
# name before the separator; `a-` or a bare `-` says nothing worth matching.
|
||
_PREFIX_MIN_STEM = 2
|
||
PREFIX_MARK = "*"
|
||
# Inside a dynamic expression: string literals (ternary arms, array items,
|
||
# quoted object keys) and the bare keys of object literals.
|
||
_STR_LIT_RE = re.compile(r"""'([^'\\]*)'|"([^"\\]*)"|`([^`]*)`""")
|
||
_OBJ_SPAN_RE = re.compile(r"\{([^{}]*)\}")
|
||
_OBJ_KEY_RE = re.compile(r"(?:^|[{,\s])([A-Za-z_][A-Za-z0-9_-]*)\s*:(?!:)")
|
||
_TEMPLATE_HOLE_RE = re.compile(r"\$\{[^}]*\}")
|
||
# A server-side / mustache interpolation inside a static value (`{{ cls }}`,
|
||
# `{% if %}`): unknowable at read time, contributes no token.
|
||
_MUSTACHE_RE = re.compile(r"\{[{%][^}]*[}%]\}")
|
||
|
||
|
||
def _class_tokens(value: str) -> list[str]:
|
||
"""The class tokens of a static attribute value: whitespace-split, only
|
||
well-formed names. An interpolation (`{{ cls }}`, `${cls}`) is blanked
|
||
before the split, so a name built around one leaves its static head —
|
||
`status-` from `status-{{ s }}` — which is emitted as the prefix
|
||
reference `status-*` rather than as a class nothing is called."""
|
||
out: list[str] = []
|
||
for t in _MUSTACHE_RE.sub(" ", value).split():
|
||
if not _CLASS_TOKEN_RE.match(t):
|
||
continue
|
||
if t.endswith(("-", "_")):
|
||
if len(t.rstrip("-_")) >= _PREFIX_MIN_STEM:
|
||
out.append(t + PREFIX_MARK)
|
||
continue
|
||
out.append(t)
|
||
return out
|
||
|
||
|
||
def _dynamic_class_tokens(expr: str) -> list[str]:
|
||
"""Class tokens named by a dynamic class expression: every string
|
||
literal's tokens (a template literal's static text only — its `${…}`
|
||
holes are unknowable) and the bare keys of object literals. Bare
|
||
identifiers elsewhere (`cond ? clsA : clsB`) are variables, not names."""
|
||
out: list[str] = []
|
||
for m in _STR_LIT_RE.finditer(expr):
|
||
literal = m.group(1) if m.group(1) is not None else (
|
||
m.group(2) if m.group(2) is not None else m.group(3)
|
||
)
|
||
if m.group(3) is not None:
|
||
literal = _TEMPLATE_HOLE_RE.sub(" ", literal)
|
||
out.extend(_class_tokens(literal))
|
||
for span in _OBJ_SPAN_RE.finditer(expr):
|
||
# Quoted keys were read as literals above; bare keys here.
|
||
body = _STR_LIT_RE.sub(" ", span.group(1))
|
||
out.extend(k for k in _OBJ_KEY_RE.findall(body) if _CLASS_TOKEN_RE.match(k))
|
||
return out
|
||
|
||
|
||
def class_references(path: str, text: str) -> dict[str, int]:
|
||
"""class token → how many times this file's markup names it. Empty for
|
||
files that carry no markup (by suffix). Reads the static `class=` /
|
||
`className=` attributes, the Vue and React dynamic forms and Svelte's
|
||
`class:x` directive; never a CSS selector (`.x {` is a definition, read
|
||
by extract_definitions) and never a script's `querySelector('.x')`.
|
||
|
||
Two forms name classes without spelling them out, and both are read
|
||
(#2970): a transition `name=` stands for every class the framework
|
||
generates from it, and a concatenated name contributes the prefix
|
||
reference `head-*` — which resolve_consumers matches against every row
|
||
whose symbol starts with `head-`."""
|
||
if not (path or "").lower().endswith(_TEMPLATE_SUFFIXES):
|
||
return {}
|
||
counts: dict[str, int] = {}
|
||
|
||
def bump(tokens: list[str]) -> None:
|
||
for t in tokens:
|
||
counts[t] = counts.get(t, 0) + 1
|
||
|
||
for m in _STATIC_CLASS_RE.finditer(text):
|
||
bump(_class_tokens(m.group(1) if m.group(1) is not None else m.group(2)))
|
||
for m in _DYNAMIC_CLASS_RE.finditer(text):
|
||
expr = next((g for g in m.groups() if g is not None), "")
|
||
bump(_dynamic_class_tokens(expr))
|
||
bump([m.group(1) for m in _SVELTE_CLASS_RE.finditer(text)])
|
||
for m in _TRANSITION_NAME_RE.finditer(text):
|
||
name = next((g for g in m.groups() if g is not None), "").strip()
|
||
if not _CLASS_TOKEN_RE.match(name):
|
||
continue
|
||
bump([name + suffix for suffix in _TRANSITION_SUFFIXES])
|
||
return counts
|
||
|
||
|
||
def extract_shapes(text: str) -> list[tuple[str, str]]:
|
||
"""Every (kind, name) this text DEFINES — kind is "css" or "sym".
|
||
|
||
Rule-for-rule mirror of the hook's awk program: first match wins per
|
||
line, dunders are skipped (every class defines __init__ — guaranteed
|
||
noise), duplicates within one text count once.
|
||
"""
|
||
return [(d.kind, d.name) for d in extract_definitions(text)]
|
||
|
||
|
||
def scannable(path: str) -> bool:
|
||
"""Should this repo file be scanned for shapes at all?"""
|
||
parts = path.split("/")
|
||
if any(p in _SKIP_DIRS for p in parts[:-1]):
|
||
return False
|
||
return not path.lower().endswith(_SKIP_SUFFIXES)
|
||
|
||
|
||
# --- the file as a unit (milestone 439) ---------------------------------------
|
||
#
|
||
# A component is the thing people reuse in a UI, and in a single-file
|
||
# component nothing inside is named after it: CatalogueBookCard.svelte defines
|
||
# `.card`, `.title` and a `Props`, never `CatalogueBookCard`. A ledger of
|
||
# definitions alone therefore could never say "a new card was built where
|
||
# CatalogueBookCard is canon". So such a file gets a row of its own.
|
||
#
|
||
# WHICH FILES, decided by structure rather than by a framework list: a file
|
||
# that RENDERS — its markup names classes, or it opens with a <script>,
|
||
# <template> or <style> block — and that DEFINES NOTHING NAMED AFTER ITSELF.
|
||
# A TSX `function CatalogueBookCard` already has its row; a module of helpers
|
||
# is accounted for by its helpers. Emitting a row for every file would put
|
||
# every module in every project into the todo at once, which is a flood, not
|
||
# a question. No body fingerprint either: every edit would otherwise ask for
|
||
# the file's judgment to be re-confirmed.
|
||
_RENDER_BLOCK_RE = re.compile(r"^\s*<(?:script|template|style)\b", re.IGNORECASE | re.MULTILINE)
|
||
|
||
|
||
def file_stem(path: str) -> str:
|
||
"""`web/src/lib/StatusChip.svelte` → `StatusChip`."""
|
||
return path.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||
|
||
|
||
def is_file_unit(
|
||
path: str, text: str, defs: list[Definition], refs: dict[str, int] | None = None,
|
||
) -> bool:
|
||
"""Is this file a candidate shape in its own right? See the block above."""
|
||
stem = file_stem(path)
|
||
if not stem:
|
||
return False
|
||
if any(d.name == stem for d in defs if d.kind == "sym"):
|
||
return False
|
||
return bool(refs) or _RENDER_BLOCK_RE.search(text) is not None
|
||
|
||
|
||
class ArchiveShape(NamedTuple):
|
||
"""A definition located in a repo archive — what the sync upserts and
|
||
the proposer matches. The leading (path, kind, name) triple is the
|
||
ledger identity; the rest is the fingerprint and the transient body."""
|
||
|
||
path: str
|
||
kind: str
|
||
name: str
|
||
signature: str
|
||
body_sha: str
|
||
body: str
|
||
scoped: bool = False # one-off by construction (#2869)
|
||
|
||
|
||
def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
|
||
"""(path, kind, name) for every definition in a repo tarball — the
|
||
identity view of definitions_from_archive."""
|
||
return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)]
|
||
|
||
|
||
class ArchiveScan(NamedTuple):
|
||
"""One walk of a repo tarball: what each file DEFINES (the ledger rows)
|
||
and which class names each file's markup REFERENCES (the CSS consumer
|
||
map, milestone 302) — read together because the bodies are in hand once."""
|
||
|
||
definitions: list[ArchiveShape]
|
||
references: dict[str, dict[str, int]] # path → class token → count
|
||
|
||
|
||
def definitions_from_archive(blob: bytes) -> list[ArchiveShape]:
|
||
"""Every definition in a repo tarball, with its fingerprint and body —
|
||
the definitions half of scan_archive."""
|
||
return scan_archive(blob).definitions
|
||
|
||
|
||
def scan_archive(blob: bytes) -> ArchiveScan:
|
||
"""Every definition in a repo tarball, with its fingerprint and body,
|
||
plus each template-bearing file's class references.
|
||
|
||
Forge archives wrap content in a single top-level directory (repo-ref/);
|
||
that component is stripped so paths match recorded snippet locations,
|
||
which are repo-relative. Non-UTF-8 files are binaries and skipped.
|
||
"""
|
||
shapes: list[ArchiveShape] = []
|
||
references: dict[str, dict[str, int]] = {}
|
||
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
|
||
for member in tar:
|
||
if not member.isfile() or "/" not in member.name:
|
||
continue
|
||
path = member.name.split("/", 1)[1]
|
||
if not path or not scannable(path) or member.size > _MAX_FILE_BYTES:
|
||
continue
|
||
handle = tar.extractfile(member)
|
||
if handle is None:
|
||
continue
|
||
try:
|
||
text = handle.read().decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
continue
|
||
defs = extract_definitions(text)
|
||
scoped = scoped_definitions(path, text, defs)
|
||
refs = class_references(path, text)
|
||
if is_file_unit(path, text, defs, refs):
|
||
shapes.append(ArchiveShape(
|
||
path, "file", file_stem(path), f"file {path}", "", "",
|
||
))
|
||
shapes.extend(
|
||
ArchiveShape(
|
||
path, d.kind, d.name, d.signature, d.body_sha, d.body,
|
||
(d.kind, d.name) in scoped,
|
||
)
|
||
for d in defs
|
||
)
|
||
if refs:
|
||
references[path] = refs
|
||
return ArchiveScan(shapes, references)
|
||
|
||
|
||
# --- matching shapes against recorded locations ------------------------------
|
||
|
||
|
||
# The covering predicate (location_covers) lives in services/shape_ledger.py
|
||
# since #2788 — the ledger's canonical marking and this module's readout are
|
||
# two consumers of ONE doctrine, and the ledger is its home.
|
||
|
||
|
||
def largest_gaps(
|
||
accounted: list[tuple[str, str, str, bool]], *, top: int = 3
|
||
) -> list[dict]:
|
||
"""The directories with the most unclassified shapes — where a
|
||
classification session should start, named the way the repo names them."""
|
||
by_dir: dict[str, dict[str, int]] = {}
|
||
for path, _kind, _name, is_accounted in accounted:
|
||
d = posixpath.dirname(path) or "(root)"
|
||
row = by_dir.setdefault(d, {"total": 0, "unclassified": 0})
|
||
row["total"] += 1
|
||
if not is_accounted:
|
||
row["unclassified"] += 1
|
||
ranked = sorted(
|
||
by_dir.items(), key=lambda kv: (-kv[1]["unclassified"], kv[0])
|
||
)
|
||
return [
|
||
{"dir": d, "unclassified": row["unclassified"], "total": row["total"]}
|
||
for d, row in ranked[:top]
|
||
if row["unclassified"]
|
||
]
|
||
|
||
|
||
# --- compute, cache, surface -------------------------------------------------
|
||
|
||
|
||
async def _recorded_locations(
|
||
user_id: int, project_id: int
|
||
) -> list[tuple[int, str, str]]:
|
||
"""(snippet_note_id, path, symbol) for every location of every live
|
||
snippet in a project — the canonical-marking input (#2788): the id is what
|
||
lets a ledger row point back at the snippet it references."""
|
||
from sqlalchemy import select
|
||
|
||
from scribe.models import async_session
|
||
from scribe.models.note import Note
|
||
from scribe.services.snippets import SNIPPET_NOTE_TYPE, snippet_fields
|
||
|
||
async with async_session() as session:
|
||
rows = await session.execute(
|
||
select(Note).where(
|
||
Note.user_id == user_id,
|
||
Note.project_id == project_id,
|
||
Note.note_type == SNIPPET_NOTE_TYPE,
|
||
Note.deleted_at.is_(None),
|
||
)
|
||
)
|
||
notes = list(rows.scalars().all())
|
||
out: list[tuple[int, str, str]] = []
|
||
for note in notes:
|
||
for loc in snippet_fields(note).get("locations") or []:
|
||
out.append(
|
||
(int(note.id), loc.get("path") or "", loc.get("symbol") or "")
|
||
)
|
||
return out
|
||
|
||
|
||
async def compute_coverage(
|
||
user_id: int, project_id: int, *, selector: ForgeSelector | None = None
|
||
) -> dict | None:
|
||
"""Sync the shape ledger from the bound repos and read the accounting.
|
||
|
||
Since #2788 this is the ledger's sync point, not just a measurement:
|
||
every walk upserts the extracted shapes (new → unclassified, vanished →
|
||
stamped), re-stamps snippet reference locations as canonical, and then
|
||
reports the ACCOUNTING — how many shapes carry a classification at all —
|
||
rather than the old "has a snippet" fraction. Unclassified is the todo
|
||
(note 2786).
|
||
|
||
None means "nothing to measure" — the owner's keyring serves none of the
|
||
project's bound repos (#2778). That is the ordinary state for a
|
||
forge-less user and every caller treats it as silence, not failure; the
|
||
ledger is untouched in that case. Forge errors (unreachable, bad token)
|
||
RAISE — the two callers are a refresh button and a background task, and
|
||
both want to know.
|
||
|
||
``user_id`` is the project OWNER's id: the cache lives there, and the
|
||
keyring resolved here must be the same one every other read uses.
|
||
"""
|
||
from scribe.services import shape_ledger
|
||
from scribe.services.forge import ForgeError
|
||
|
||
if selector is None:
|
||
selector = await get_forges(user_id, project_id)
|
||
if not selector.configured:
|
||
return None
|
||
|
||
served: list[tuple[str, str]] = []
|
||
recorded = await _recorded_locations(user_id, project_id)
|
||
# The proposer's canon catalog, read once per refresh and shared across
|
||
# the project's repos (#2792).
|
||
canons = None
|
||
proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0}
|
||
for binding in await bindings_for_project(user_id, project_id):
|
||
key = binding.repo_key
|
||
hit = selector.resolve(key)
|
||
if hit is None:
|
||
continue # bound to a host no connection serves
|
||
forge, api_repo = hit
|
||
# The binding's own ref when it names one (#2873: a dev-first project
|
||
# has its ledger follow dev), else the forge's default branch.
|
||
ref = binding.ref or await forge.default_branch(api_repo)
|
||
scan = scan_archive(await forge.archive(api_repo, ref))
|
||
definitions = scan.definitions
|
||
# The head commit is provenance sugar on the ledger rows; failing to
|
||
# learn it must not fail the sync — the ref names the point well
|
||
# enough and the row timestamps carry the when.
|
||
try:
|
||
marker = await forge.latest_commit(api_repo, "", ref) or ref
|
||
except ForgeError:
|
||
marker = ref
|
||
await shape_ledger.sync_repo_shapes(
|
||
project_id, key, definitions, seen_marker=marker
|
||
)
|
||
served.append((key, ref))
|
||
# The CSS consumer map (milestone 302) rides the same archive: which
|
||
# files' markup names each class. Mechanical and recomputable, so it
|
||
# must not be able to fail the refresh either.
|
||
try:
|
||
await shape_ledger.sync_repo_consumers(project_id, key, scan.references)
|
||
except Exception:
|
||
logger.warning("consumer map sync failed for %s", key, exc_info=True)
|
||
# Propose while the bodies are in hand — the one moment they exist.
|
||
# Canonical marking below only touches rows the proposer leaves
|
||
# alone (a canon's own location never gets a proposal), so the order
|
||
# is immaterial; the proposer must not be able to fail the refresh.
|
||
try:
|
||
if canons is None:
|
||
canons = await shape_ledger.canon_catalog(user_id)
|
||
stats = await shape_ledger.propose_for_repo(
|
||
user_id, project_id, key, definitions, canons=canons
|
||
)
|
||
for k in proposer_stats:
|
||
proposer_stats[k] += stats.get(k, 0)
|
||
except Exception:
|
||
logger.warning("shape proposer failed for %s", key, exc_info=True)
|
||
if not served:
|
||
return None
|
||
|
||
await shape_ledger.mark_canonicals(project_id, recorded)
|
||
try:
|
||
await shape_ledger.apply_derive_groups(project_id)
|
||
except Exception:
|
||
logger.warning("derive-first grouping failed", exc_info=True)
|
||
# "Since the previous computation" — the cache's stamp. A first seed has
|
||
# none, so nothing is new then. Read once; two passes use it: the
|
||
# button-B flag (#2793) and the derive-new drift count (#2899).
|
||
since = None
|
||
try:
|
||
previous = await get_setting(user_id, f"{_CACHE_KEY_PREFIX}{project_id}")
|
||
if previous:
|
||
stamp = (json.loads(previous) or {}).get("computed_at")
|
||
since = datetime.fromisoformat(stamp) if stamp else None
|
||
except Exception:
|
||
logger.warning("previous coverage stamp unreadable", exc_info=True)
|
||
# The button-B pass (#2793): shapes new since the PREVIOUS computation,
|
||
# where a canon dominates.
|
||
try:
|
||
await shape_ledger.flag_divergence(project_id, since=since)
|
||
except Exception:
|
||
logger.warning("divergence pass failed", exc_info=True)
|
||
|
||
# Project-wide readout, deliberately wider than this walk: a second bound
|
||
# repo that was unreachable today still has live rows, and they count.
|
||
rows = await shape_ledger.live_rows(project_id)
|
||
counts = {"canonical": 0, "instance": 0, "variant": 0, "exempt": 0,
|
||
"scoped": 0, "unclassified": 0}
|
||
for row in rows:
|
||
counts[row.status] = counts.get(row.status, 0) + 1
|
||
by_repo: dict[str, dict[str, int]] = {}
|
||
for row in rows:
|
||
agg = by_repo.setdefault(row.repo_key, {"total": 0, "accounted": 0})
|
||
agg["total"] += 1
|
||
agg["accounted"] += row.status != "unclassified"
|
||
|
||
unclassified = counts.pop("unclassified")
|
||
# The CSS consumer map's readout (milestone 302): which files render each
|
||
# css row — on the derive groups (a shared recipe vs a scoped one is a
|
||
# count), and the negative space: css rules no template names. "Unused"
|
||
# is measured only where the map has evidence of templates at all (one
|
||
# edge somewhere); a repo of bare stylesheets is "not measured", not
|
||
# "all unused".
|
||
css_rows = [r for r in rows if r.kind == "css"]
|
||
consumer_paths: dict[int, list[str]] = {}
|
||
unused_css = None
|
||
try:
|
||
edges = await shape_ledger.consumers_of([r.id for r in css_rows])
|
||
consumer_paths = {sid: [e.path for e in es] for sid, es in edges.items()}
|
||
if consumer_paths:
|
||
unused_css = sum(1 for r in css_rows if r.id not in consumer_paths)
|
||
except Exception:
|
||
logger.warning("consumer map read failed", exc_info=True)
|
||
proposals = shape_ledger.proposal_summary(rows, consumer_paths=consumer_paths)
|
||
divergence = shape_ledger.divergence_summary(rows)
|
||
derive_new = shape_ledger.derive_new_summary(rows, since=since)
|
||
# What `stamps_to_review` would list (#4608). The review tool existed and
|
||
# nobody on two projects knew to call it: 132 and 488 weak stamps sat
|
||
# electing canons for weeks. The arrival line is the one place a
|
||
# project's own session is sure to look, so the count goes there.
|
||
try:
|
||
review = shape_ledger.stamps_to_review(rows, top=0)
|
||
except Exception:
|
||
logger.warning("stamp review read failed", exc_info=True)
|
||
review = {}
|
||
# The write-time figure (milestone 439): how often the end-of-turn
|
||
# question was put on this project, and how often it was walked past.
|
||
# An audit reads the ledger; this reads whether the reflex held.
|
||
try:
|
||
from scribe.services import shape_check as shape_check_svc
|
||
|
||
write_time = await shape_check_svc.window_summary(project_id)
|
||
except Exception:
|
||
logger.warning("write-time figure read failed", exc_info=True)
|
||
write_time = None
|
||
return {
|
||
"total": len(rows),
|
||
"accounted": len(rows) - unclassified,
|
||
"unclassified": unclassified,
|
||
"counts": counts,
|
||
# The proposer's standing (#2792): canon proposals awaiting a
|
||
# confirm, the largest derive-first groups, and what this refresh did.
|
||
"proposed": proposals["proposed"],
|
||
"derive_groups": proposals["derive_groups"],
|
||
"top_canon": proposals.get("top_canon"),
|
||
# Drift since the previous refresh (#2899): copies that joined a
|
||
# duplicate family — what the arrival line names so drift is noticed
|
||
# on entering, not found by an audit.
|
||
"derive_new": derive_new,
|
||
# The consumer map's negative space (milestone 302): live css rules
|
||
# no template names — None when the map has no evidence of templates.
|
||
"unused_css": unused_css,
|
||
"proposer": proposer_stats,
|
||
# The divergence readout (#2793): button B where button A is canon,
|
||
# and judged shapes whose bodies moved since they were judged.
|
||
"divergent": divergence["divergent"],
|
||
"divergence": divergence["divergence"],
|
||
"recheck": divergence["recheck"],
|
||
# Unattended stamps below today's floor, and canons whose members no
|
||
# longer agree what they are — the judgment queue `stamps_to_review`
|
||
# holds (#4608).
|
||
"weak_stamps": review.get("weak_count", 0),
|
||
"incoherent_canons": review.get("incoherent_count", 0),
|
||
# {checked, asked, left, days} from the Stop hook's recorded outcomes.
|
||
"write_time": write_time,
|
||
# Honesty flag, not decoration: every surface that shows the number
|
||
# is expected to carry it through.
|
||
"estimate": True,
|
||
"computed_at": datetime.now(timezone.utc).isoformat(),
|
||
"repos": [
|
||
{"repo": key, "ref": ref,
|
||
**by_repo.get(key, {"total": 0, "accounted": 0})}
|
||
for key, ref in served
|
||
],
|
||
"largest_gaps": largest_gaps([
|
||
(r.path, r.kind, r.symbol, r.status != "unclassified")
|
||
for r in rows
|
||
]),
|
||
}
|
||
|
||
|
||
async def refresh_coverage(
|
||
user_id: int, project_id: int, *, selector: ForgeSelector | None = None
|
||
) -> dict | None:
|
||
"""Compute and cache. The only writer of the cache key."""
|
||
coverage = await compute_coverage(user_id, project_id, selector=selector)
|
||
if coverage is not None:
|
||
await set_setting(
|
||
user_id, f"{_CACHE_KEY_PREFIX}{project_id}", json.dumps(coverage)
|
||
)
|
||
return coverage
|
||
|
||
|
||
# The arrival-moment self-seed (#2802). A ledger that has never been computed
|
||
# used to wait for someone to find the UI Refresh button; entering the project
|
||
# is the moment the number is wanted, so entering is the moment it seeds.
|
||
_SEED_MAX_AGE = timedelta(hours=24)
|
||
|
||
# Projects with a refresh already running — concurrent enters must not fetch
|
||
# the same tarball N times. In-process on purpose: the cost being bounded is
|
||
# per-process forge traffic, and a rare double-fetch across workers is
|
||
# harmless (the upsert is idempotent).
|
||
_inflight_seed: set[int] = set()
|
||
|
||
|
||
async def refresh_if_stale(
|
||
user_id: int, project_id: int, cached: dict | None = None
|
||
) -> None:
|
||
"""Background-refresh a project's ledger when its readout is absent or
|
||
older than a day. Fire-and-forget material (background.spawn): every exit
|
||
is quiet, every failure a WARNING — a seed must never surface as an
|
||
enter_project error. ``user_id`` is the project OWNER (whose keyring and
|
||
cache this is); pass ``cached`` when the caller already read it.
|
||
"""
|
||
try:
|
||
if cached is None:
|
||
cached = await cached_coverage(user_id, project_id)
|
||
if cached:
|
||
try:
|
||
computed = datetime.fromisoformat(cached.get("computed_at") or "")
|
||
if datetime.now(timezone.utc) - computed < _SEED_MAX_AGE:
|
||
return
|
||
except ValueError:
|
||
pass # an unreadable stamp reads as stale
|
||
if project_id in _inflight_seed:
|
||
return
|
||
selector = await get_forges(user_id, project_id)
|
||
if not selector.configured:
|
||
return # rule #115: forge-less stays exactly as it was
|
||
_inflight_seed.add(project_id)
|
||
try:
|
||
await refresh_coverage(user_id, project_id, selector=selector)
|
||
finally:
|
||
_inflight_seed.discard(project_id)
|
||
except Exception:
|
||
logger.warning(
|
||
"background coverage seed failed for project %s", project_id,
|
||
exc_info=True,
|
||
)
|
||
|
||
|
||
async def refresh_for_caller(caller_id: int, project_id: int) -> dict:
|
||
"""The explicit agent-facing refresh (#2802) — synchronous, named errors.
|
||
|
||
Write-gated: recomputing spends the owner's forge API budget and rewrites
|
||
ledger rows' seen-markers, so a read share doesn't grant it. Resolution
|
||
runs on the OWNER's keyring like every other forge read. Raises
|
||
ValueError with a fixable message instead of silently measuring nothing —
|
||
the caller is an agent mid-task, and "None" would strand it exactly the
|
||
way the button-only path did.
|
||
"""
|
||
from scribe.models import async_session
|
||
from scribe.models.project import Project
|
||
from scribe.services import access
|
||
|
||
if not await access.can_write_project(caller_id, project_id):
|
||
raise ValueError(f"project {project_id} not found or no write access")
|
||
async with async_session() as session:
|
||
project = await session.get(Project, project_id)
|
||
if project is None:
|
||
raise ValueError(f"project {project_id} not found")
|
||
owner_id = project.user_id or caller_id
|
||
selector = await get_forges(owner_id, project_id)
|
||
if not selector.configured:
|
||
raise ValueError(
|
||
"No forge connection serves this project — the owner adds one "
|
||
"under Settings → Integrations → Git Forges"
|
||
)
|
||
coverage = await refresh_coverage(owner_id, project_id, selector=selector)
|
||
if coverage is None:
|
||
raise ValueError(
|
||
"No bound repo is served by the owner's forge connections — "
|
||
"bind_repo the project's repo on a host a connection serves"
|
||
)
|
||
return coverage
|
||
|
||
|
||
async def cached_coverage(user_id: int, project_id: int) -> dict | None:
|
||
"""The last computed summary, or None — never computes."""
|
||
raw = await get_setting(user_id, f"{_CACHE_KEY_PREFIX}{project_id}", "")
|
||
if not raw:
|
||
return None
|
||
try:
|
||
parsed = json.loads(raw)
|
||
except ValueError:
|
||
return None
|
||
return parsed if isinstance(parsed, dict) else None
|
||
|
||
|
||
def coverage_line(coverage: dict) -> str:
|
||
"""The one-line evidence-carrying summary enter_project surfaces."""
|
||
day = (coverage.get("computed_at") or "")[:10]
|
||
counts = coverage.get("counts") or {}
|
||
breakdown = " · ".join(
|
||
f"{counts[k]} {k}"
|
||
for k in ("canonical", "instance", "variant", "exempt", "scoped")
|
||
if counts.get(k)
|
||
)
|
||
line = (
|
||
f"shape accounting: {coverage.get('accounted', 0)}"
|
||
f"/{coverage.get('total', 0)} shapes accounted for"
|
||
)
|
||
if breakdown:
|
||
line += f" — {breakdown}"
|
||
line += f" (estimate{', computed ' + day if day else ''})"
|
||
unclassified = coverage.get("unclassified", 0)
|
||
# The standing work, built whatever the todo count (#2899). Since the
|
||
# scoped bucket (#2869) a ledger can read 100% accounted and still carry
|
||
# derive groups, proposals and divergence; gating this block on
|
||
# `unclassified > 0` is how 439 derive rows went unmentioned.
|
||
standing = []
|
||
if coverage.get("proposed"):
|
||
standing.append(f"{coverage['proposed']} proposed")
|
||
n_groups = len(coverage.get("derive_groups") or [])
|
||
if n_groups:
|
||
standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}")
|
||
# Drift since the previous refresh: copies that joined a family, the
|
||
# first one named — the sentence the arrival moment exists to say.
|
||
new = coverage.get("derive_new") or {}
|
||
if new.get("count"):
|
||
n = new["count"]
|
||
first_new = (new.get("examples") or [{}])[0]
|
||
where = (
|
||
f": {first_new['label']} in {first_new['path']}"
|
||
if first_new.get("label") and first_new.get("path") else ""
|
||
)
|
||
standing.append(f"+{n} new cop{'y' if n == 1 else 'ies'} since last refresh{where}")
|
||
if coverage.get("divergent"):
|
||
standing.append(f"{coverage['divergent']} DIVERGENT")
|
||
# The next action, on the line (#2874): the canon with the biggest
|
||
# queue to confirm, and the widest body-identical copy to consolidate.
|
||
top = coverage.get("top_canon") or {}
|
||
if top.get("snippet_id"):
|
||
standing.append(f"top canon #{top['snippet_id']} ×{top.get('count', 0)}")
|
||
first = (coverage.get("derive_groups") or [{}])[0]
|
||
if first.get("label") and first.get("files"):
|
||
top_copy = f"top copy {first['label']} ×{first['files']} files"
|
||
# A css family says what renders it (milestone 302): the count that
|
||
# tells a shared recipe from a scoped convention.
|
||
if "consumers" in first:
|
||
n_t = (first.get("consumers") or {}).get("count", 0)
|
||
top_copy += f" · used by {n_t} template{'s' if n_t != 1 else ''}"
|
||
standing.append(top_copy)
|
||
if coverage.get("unused_css"):
|
||
n_u = coverage["unused_css"]
|
||
standing.append(f"{n_u} unused class{'es' if n_u != 1 else ''}")
|
||
if unclassified:
|
||
line += f"; {unclassified} unclassified"
|
||
if standing:
|
||
line += f" ({', '.join(standing)})"
|
||
gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
|
||
if gaps:
|
||
line += ", largest: " + ", ".join(gaps)
|
||
elif standing:
|
||
line += f"; standing: {', '.join(standing)}"
|
||
if coverage.get("recheck"):
|
||
line += f"; {coverage['recheck']} judged shape{'s' if coverage['recheck'] != 1 else ''} changed since judged — recheck"
|
||
# Rows the hook asserted on evidence the ledger no longer accepts (#4608).
|
||
# They no longer elect a canon, but they still claim to be instances
|
||
# until someone reads them — the tool to call is named, not implied.
|
||
review = []
|
||
if coverage.get("weak_stamps"):
|
||
n_w = coverage["weak_stamps"]
|
||
review.append(f"{n_w} weak stamp{'s' if n_w != 1 else ''}")
|
||
if coverage.get("incoherent_canons"):
|
||
n_i = coverage["incoherent_canons"]
|
||
review.append(f"{n_i} incoherent canon{'s' if n_i != 1 else ''}")
|
||
if review:
|
||
line += f"; {' · '.join(review)} to judge — stamps_to_review"
|
||
# Whether the writers judged what they wrote (milestone 439). Silent until
|
||
# the question has been put at least once: a project nobody has written to
|
||
# through the plugin has no figure, not a perfect one.
|
||
wt = coverage.get("write_time") or {}
|
||
if wt.get("checked"):
|
||
line += (
|
||
f"; written-shape check ({wt.get('days', 7)}d): {wt['checked']} "
|
||
f"turn{'s' if wt['checked'] != 1 else ''} checked, {wt.get('asked', 0)} asked, "
|
||
f"{wt.get('left', 0)} left unjudged"
|
||
)
|
||
return line
|