"""Pattern-library coverage — what fraction of a bound repo's shapes have a recorded snippet (#2692, forge job 3 of decision #2686). The all-shapes doctrine says every shape gets recorded at first build. This module is the hoping→knowing move: it enumerates the definitions that exist in a project's bound repos (via the forge, one archive download per repo) and compares them against recorded snippet locations, so "record everything" becomes a watched number instead of an aspiration. The definition extractor MIRRORS the write-path hook's awk rules (plugin/hooks/scribe_prior_art.sh, ARM 1) — one shared notion of "a definition" between the hook and the server, so the metric and the backstop agree on what counts. The two are pinned together by shared test vectors in tests/test_pattern_coverage.py; change one, change both. The number is an ESTIMATE and every surface must say so: keyword extraction over-counts (private one-offs, generated code that slips the dir filter) and under-counts (keyword-less declaration syntax — C/Java/Dart — needs a real parser and is out of scope, exactly as it is for the hook). The trend carries the meaning, like the usage counters; the raw number is not a grade. Compute is on demand + cached with a freshness stamp — recomputed on webhook push and explicit refresh, NEVER in the request path of enter_project, which only ever reads the cache. """ from __future__ import annotations import hashlib import io import json import logging import posixpath import re import tarfile from typing import NamedTuple from datetime import datetime, timedelta, timezone from scribe.services.forge import ForgeSelector, get_forges from scribe.services.repo_bindings import bindings_for_project from scribe.services.settings import get_setting, set_setting logger = logging.getLogger(__name__) # Cache key in the settings KV, on the project OWNER's user_id — the same # channel the scheduler's last-run summary uses for machine-written state. # v2 suffix with #2788: the payload shape inverted (accounted/unclassified); # pre-ledger blobs under the old key simply stop being found, so the card # honestly reads "not measured yet" until the first ledger-era refresh. _CACHE_KEY_PREFIX = "pattern_coverage_v2_" # Files whose content can't hold definitions — the hook's skip list, verbatim, # plus sourcemaps (which are JSON in a trenchcoat). _SKIP_SUFFIXES = ( ".md", ".mdx", ".txt", ".rst", ".json", ".lock", ".log", ".csv", ".tsv", ".svg", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".map", ) # Vendored/generated trees would swamp the metric with shapes nobody should # record — the dunder-skip lesson at directory scale: guaranteed noise teaches # people to ignore the number. _SKIP_DIRS = frozenset({ "node_modules", "vendor", "dist", "build", "target", "__pycache__", ".git", ".venv", "venv", }) # A single source file bigger than this is almost certainly generated or # vendored (bundles, lockstep protos) — skipped, and part of why the number # is labeled an estimate. _MAX_FILE_BYTES = 1_000_000 # --- the definition extractor (mirror of scribe_prior_art.sh ARM 1) ---------- _CSS_RE = re.compile(r"^\s*\.([A-Za-z][A-Za-z0-9_-]*)\s*[,{]") # Leading declaration modifiers, so the definition keyword is the first word # regardless of language (export/pub/private/suspend/...). _MODIFIERS_RE = re.compile( r"^(?:(?:pub(?:\([a-z]+\))?|export|default|private|internal|protected" r"|public|static|suspend|async|open|sealed|data|abstract|final|inline" r"|unsafe|extern|override)\s+)*" ) # Go method with receiver: func (r *T) Name( _GO_METHOD_RE = re.compile(r"^func\s*\([^)]*\)\s*([A-Za-z_][A-Za-z0-9_]*)") # Keyword-announced definitions, functions and named types alike. `impl` is # excluded on purpose — several per type is normal Rust, not duplication. _KEYWORD_RE = re.compile( r"^(?:function|def|class|func|fun|fn|sub|struct|trait|interface|enum" r"|object|protocol|type)\s+([A-Za-z_$][A-Za-z0-9_$]*)" ) # Arrow/expression assignment: const name = (…) / let name = async ( _ARROW_RE = re.compile( r"^(?:const|let)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?[(<]" ) class Definition(NamedTuple): """One extracted definition with its content fingerprint (#2792). `signature` is the definition line itself; `body_sha` hashes the block whitespace- and comment-insensitively; `body` is the block's text, held only for the duration of a refresh (the proposer matches on it) and never stored. """ kind: str name: str signature: str body_sha: str body: str line: int = -1 # 0-based line the definition starts on (#2869) def _definition_on(raw: str) -> tuple[str, str] | None: """The (kind, name) this one line defines, or None. First match wins — the same order the hook's awk program tries.""" m = _CSS_RE.match(raw) if m: return ("css", m.group(1)) line = _MODIFIERS_RE.sub("", raw.lstrip()) if m := _GO_METHOD_RE.match(line): return ("sym", m.group(1)) if m := _KEYWORD_RE.match(line): name = m.group(1) if name.startswith("__") and name.endswith("__"): return None # `type` announces a definition only when something is declared after # the name (`type Foo = …`, `type Foo struct {`); an import specifier # (`import { type Foo, bar }`) is the same two words and defines # nothing — it showed up as a two-file "identical body" family (#2904). if line.startswith("type") and not re.search(r"[={]", line[m.end():]): return None return ("sym", name) if m := _ARROW_RE.match(line): return ("sym", m.group(1)) return None # A definition's block runs from its line until the next non-blank line at # its own indentation or shallower that is not a closer — so a Python def ends # at the next top-level statement, a braces block keeps its `}`, a CSS rule # keeps its `}`. Capped so a generated monolith can't make one shape's # fingerprint cover the file. _BLOCK_CAP = 120 _CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);") # Lines that don't change what a shape IS: comments and decorators. Dropped # from the fingerprint so touching a comment above the next function doesn't # read as this one's body changing. _NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "") _SIGNATURE_CAP = 300 def _indent(line: str) -> int: return len(line) - len(line.lstrip()) def _block_sha(lines: list[str]) -> str: kept = [ " ".join(ln.split()) for ln in lines if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES) ] return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16] def _declaration_count(lines: list[str]) -> int: """How many `prop: value` declarations a CSS block body carries.""" body = " ".join(lines) return sum(1 for part in body.replace("}", "").split(";") if ":" in part) def extract_definitions(text: str) -> list[Definition]: """Every definition this text makes, with signature + fingerprint. Duplicate (kind, name) within one text collapse to the first — the ledger's identity is per file, so a second definition of the same name (an overload, a re-declaration) is the same shape to it. """ lines = text.splitlines() starts: list[tuple[int, str, str]] = [] for i, raw in enumerate(lines): hit = _definition_on(raw) if hit: starts.append((i, hit[0], hit[1])) seen: set[tuple[str, str]] = set() out: list[Definition] = [] for i, kind, name in starts: if (kind, name) in seen: continue seen.add((kind, name)) base = _indent(lines[i]) end = min(len(lines), i + _BLOCK_CAP) for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)): ln = lines[j] if not ln.strip(): continue if _indent(ln) <= base and ln.strip() not in _CLOSERS: end = j break block = lines[i:end] # A CSS rule's fingerprint is its DECLARATIONS, not its selector # (#2872): the row's identity already carries the selector, and the # question the fingerprint answers for derive grouping is "is this the # same rule under another name?" — .closed-msg / .error-block / # .success-msg with identical bodies are one dup group, not three # lonely rows. Sym blocks keep their signature line in the hash. if kind == "css": # One-line rules (`.x { color: red; }`) carry their declarations on # the selector line itself; a block that is only the selector plus # trailing blanks must not hash to the empty string (which grouped # 68 unrelated one-liners as one "copy" on first deploy, #2872). first = lines[i] brace = first.find("{") head = [first[brace + 1:]] if brace >= 0 and first[brace + 1:].strip() else [] hashed = head + block[1:] if not any(x.strip() for x in hashed): hashed = block # A SINGLE declaration is not a shape (#2903): `color: var(--fs- # text-tertiary)` under .text-muted, .task-mark and .pin-badge-auto # is three meanings sharing one line, not three copies of one # rule — the first pay-down found 5-file "families" of exactly # this and nobody would consolidate them. Keep the selector in the # hash for one-liners, so they group only with same-name copies; # two declarations and up stay selector-agnostic. elif _declaration_count(hashed) < 2: hashed = block else: hashed = block out.append(Definition( kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(hashed), "\n".join(block), i, )) return out # --- by-construction scope (#2869) ------------------------------------------- # # A Vue single-file component's `