FabledGrimoire/src/scribe/services/coverage.py

"""Pattern-library coverage — what fraction of a bound repo's shapes have a
recorded snippet (#2692, forge job 3 of decision #2686).

The all-shapes doctrine says every shape gets recorded at first build. This
module is the hoping→knowing move: it enumerates the definitions that exist in
a project's bound repos (via the forge, one archive download per repo) and
compares them against recorded snippet locations, so "record everything"
becomes a watched number instead of an aspiration.

The definition extractor MIRRORS the write-path hook's awk rules
(plugin/hooks/scribe_prior_art.sh, ARM 1) — one shared notion of "a
definition" between the hook and the server, so the metric and the backstop
agree on what counts. The two are pinned together by shared test vectors in
tests/test_pattern_coverage.py; change one, change both.

The number is an ESTIMATE and every surface must say so: keyword extraction
over-counts (private one-offs, generated code that slips the dir filter) and
under-counts (keyword-less declaration syntax — C/Java/Dart — needs a real
parser and is out of scope, exactly as it is for the hook). The trend carries
the meaning, like the usage counters; the raw number is not a grade.

Compute is on demand + cached with a freshness stamp — recomputed on webhook
push and explicit refresh, NEVER in the request path of enter_project, which
only ever reads the cache.
"""
from __future__ import annotations

import hashlib
import io
import json
import logging
import posixpath
import re
import tarfile
from typing import NamedTuple
from datetime import datetime, timedelta, timezone

from scribe.services.forge import ForgeSelector, get_forges
from scribe.services.repo_bindings import keys_for_project
from scribe.services.settings import get_setting, set_setting

logger = logging.getLogger(__name__)

# Cache key in the settings KV, on the project OWNER's user_id — the same
# channel the scheduler's last-run summary uses for machine-written state.
# v2 suffix with #2788: the payload shape inverted (accounted/unclassified);
# pre-ledger blobs under the old key simply stop being found, so the card
# honestly reads "not measured yet" until the first ledger-era refresh.
_CACHE_KEY_PREFIX = "pattern_coverage_v2_"

# Files whose content can't hold definitions — the hook's skip list, verbatim,
# plus sourcemaps (which are JSON in a trenchcoat).
_SKIP_SUFFIXES = (
    ".md", ".mdx", ".txt", ".rst", ".json", ".lock", ".log", ".csv", ".tsv",
    ".svg", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".map",
)

# Vendored/generated trees would swamp the metric with shapes nobody should
# record — the dunder-skip lesson at directory scale: guaranteed noise teaches
# people to ignore the number.
_SKIP_DIRS = frozenset({
    "node_modules", "vendor", "dist", "build", "target",
    "__pycache__", ".git", ".venv", "venv",
})

# A single source file bigger than this is almost certainly generated or
# vendored (bundles, lockstep protos) — skipped, and part of why the number
# is labeled an estimate.
_MAX_FILE_BYTES = 1_000_000


# --- the definition extractor (mirror of scribe_prior_art.sh ARM 1) ----------

_CSS_RE = re.compile(r"^\s*\.([A-Za-z][A-Za-z0-9_-]*)\s*[,{]")
# Leading declaration modifiers, so the definition keyword is the first word
# regardless of language (export/pub/private/suspend/...).
_MODIFIERS_RE = re.compile(
    r"^(?:(?:pub(?:\([a-z]+\))?|export|default|private|internal|protected"
    r"|public|static|suspend|async|open|sealed|data|abstract|final|inline"
    r"|unsafe|extern|override)\s+)*"
)
# Go method with receiver: func (r *T) Name(
_GO_METHOD_RE = re.compile(r"^func\s*\([^)]*\)\s*([A-Za-z_][A-Za-z0-9_]*)")
# Keyword-announced definitions, functions and named types alike. `impl` is
# excluded on purpose — several per type is normal Rust, not duplication.
_KEYWORD_RE = re.compile(
    r"^(?:function|def|class|func|fun|fn|sub|struct|trait|interface|enum"
    r"|object|protocol|type)\s+([A-Za-z_$][A-Za-z0-9_$]*)"
)
# Arrow/expression assignment: const name = (…) / let name = async (
_ARROW_RE = re.compile(
    r"^(?:const|let)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?[(<]"
)


class Definition(NamedTuple):
    """One extracted definition with its content fingerprint (#2792).

    `signature` is the definition line itself; `body_sha` hashes the block
    whitespace- and comment-insensitively; `body` is the block's text, held
    only for the duration of a refresh (the proposer matches on it) and
    never stored.
    """

    kind: str
    name: str
    signature: str
    body_sha: str
    body: str


def _definition_on(raw: str) -> tuple[str, str] | None:
    """The (kind, name) this one line defines, or None. First match wins —
    the same order the hook's awk program tries."""
    m = _CSS_RE.match(raw)
    if m:
        return ("css", m.group(1))
    line = _MODIFIERS_RE.sub("", raw.lstrip())
    if m := _GO_METHOD_RE.match(line):
        return ("sym", m.group(1))
    if m := _KEYWORD_RE.match(line):
        name = m.group(1)
        if name.startswith("__") and name.endswith("__"):
            return None
        return ("sym", name)
    if m := _ARROW_RE.match(line):
        return ("sym", m.group(1))
    return None


# A definition's block runs from its line until the next non-blank line at
# its own indentation or shallower that is not a closer — so a Python def ends
# at the next top-level statement, a braces block keeps its `}`, a CSS rule
# keeps its `}`. Capped so a generated monolith can't make one shape's
# fingerprint cover the file.
_BLOCK_CAP = 120
_CLOSERS = ("}", ")", "]", "end", "};", "});", ");", "})", "]);")
# Lines that don't change what a shape IS: comments and decorators. Dropped
# from the fingerprint so touching a comment above the next function doesn't
# read as this one's body changing.
_NOISE_PREFIXES = ("#", "//", "/*", "*", "*/", "@", "<!--", "-->")
_SIGNATURE_CAP = 300


def _indent(line: str) -> int:
    return len(line) - len(line.lstrip())


def _block_sha(lines: list[str]) -> str:
    kept = [
        " ".join(ln.split())
        for ln in lines
        if ln.strip() and not ln.lstrip().startswith(_NOISE_PREFIXES)
    ]
    return hashlib.sha1("\n".join(kept).encode("utf-8")).hexdigest()[:16]


def extract_definitions(text: str) -> list[Definition]:
    """Every definition this text makes, with signature + fingerprint.

    Duplicate (kind, name) within one text collapse to the first — the
    ledger's identity is per file, so a second definition of the same name
    (an overload, a re-declaration) is the same shape to it.
    """
    lines = text.splitlines()
    starts: list[tuple[int, str, str]] = []
    for i, raw in enumerate(lines):
        hit = _definition_on(raw)
        if hit:
            starts.append((i, hit[0], hit[1]))
    seen: set[tuple[str, str]] = set()
    out: list[Definition] = []
    for i, kind, name in starts:
        if (kind, name) in seen:
            continue
        seen.add((kind, name))
        base = _indent(lines[i])
        end = min(len(lines), i + _BLOCK_CAP)
        for j in range(i + 1, min(len(lines), i + _BLOCK_CAP)):
            ln = lines[j]
            if not ln.strip():
                continue
            if _indent(ln) <= base and ln.strip() not in _CLOSERS:
                end = j
                break
        block = lines[i:end]
        out.append(Definition(
            kind, name, lines[i].strip()[:_SIGNATURE_CAP], _block_sha(block),
            "\n".join(block),
        ))
    return out


def extract_shapes(text: str) -> list[tuple[str, str]]:
    """Every (kind, name) this text DEFINES — kind is "css" or "sym".

    Rule-for-rule mirror of the hook's awk program: first match wins per
    line, dunders are skipped (every class defines __init__ — guaranteed
    noise), duplicates within one text count once.
    """
    return [(d.kind, d.name) for d in extract_definitions(text)]


def scannable(path: str) -> bool:
    """Should this repo file be scanned for shapes at all?"""
    parts = path.split("/")
    if any(p in _SKIP_DIRS for p in parts[:-1]):
        return False
    return not path.lower().endswith(_SKIP_SUFFIXES)


class ArchiveShape(NamedTuple):
    """A definition located in a repo archive — what the sync upserts and
    the proposer matches. The leading (path, kind, name) triple is the
    ledger identity; the rest is the fingerprint and the transient body."""

    path: str
    kind: str
    name: str
    signature: str
    body_sha: str
    body: str


def shapes_from_archive(blob: bytes) -> list[tuple[str, str, str]]:
    """(path, kind, name) for every definition in a repo tarball — the
    identity view of definitions_from_archive."""
    return [(d.path, d.kind, d.name) for d in definitions_from_archive(blob)]


def definitions_from_archive(blob: bytes) -> list[ArchiveShape]:
    """Every definition in a repo tarball, with its fingerprint and body.

    Forge archives wrap content in a single top-level directory (repo-ref/);
    that component is stripped so paths match recorded snippet locations,
    which are repo-relative. Non-UTF-8 files are binaries and skipped.
    """
    shapes: list[ArchiveShape] = []
    with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
        for member in tar:
            if not member.isfile() or "/" not in member.name:
                continue
            path = member.name.split("/", 1)[1]
            if not path or not scannable(path) or member.size > _MAX_FILE_BYTES:
                continue
            handle = tar.extractfile(member)
            if handle is None:
                continue
            try:
                text = handle.read().decode("utf-8")
            except UnicodeDecodeError:
                continue
            shapes.extend(
                ArchiveShape(path, d.kind, d.name, d.signature, d.body_sha, d.body)
                for d in extract_definitions(text)
            )
    return shapes


# --- matching shapes against recorded locations ------------------------------


# The covering predicate (location_covers) lives in services/shape_ledger.py
# since #2788 — the ledger's canonical marking and this module's readout are
# two consumers of ONE doctrine, and the ledger is its home.


def largest_gaps(
    accounted: list[tuple[str, str, str, bool]], *, top: int = 3
) -> list[dict]:
    """The directories with the most unclassified shapes — where a
    classification session should start, named the way the repo names them."""
    by_dir: dict[str, dict[str, int]] = {}
    for path, _kind, _name, is_accounted in accounted:
        d = posixpath.dirname(path) or "(root)"
        row = by_dir.setdefault(d, {"total": 0, "unclassified": 0})
        row["total"] += 1
        if not is_accounted:
            row["unclassified"] += 1
    ranked = sorted(
        by_dir.items(), key=lambda kv: (-kv[1]["unclassified"], kv[0])
    )
    return [
        {"dir": d, "unclassified": row["unclassified"], "total": row["total"]}
        for d, row in ranked[:top]
        if row["unclassified"]
    ]


# --- compute, cache, surface -------------------------------------------------


async def _recorded_locations(
    user_id: int, project_id: int
) -> list[tuple[int, str, str]]:
    """(snippet_note_id, path, symbol) for every location of every live
    snippet in a project — the canonical-marking input (#2788): the id is what
    lets a ledger row point back at the snippet it references."""
    from sqlalchemy import select

    from scribe.models import async_session
    from scribe.models.note import Note
    from scribe.services.snippets import SNIPPET_NOTE_TYPE, snippet_fields

    async with async_session() as session:
        rows = await session.execute(
            select(Note).where(
                Note.user_id == user_id,
                Note.project_id == project_id,
                Note.note_type == SNIPPET_NOTE_TYPE,
                Note.deleted_at.is_(None),
            )
        )
        notes = list(rows.scalars().all())
    out: list[tuple[int, str, str]] = []
    for note in notes:
        for loc in snippet_fields(note).get("locations") or []:
            out.append(
                (int(note.id), loc.get("path") or "", loc.get("symbol") or "")
            )
    return out


async def compute_coverage(
    user_id: int, project_id: int, *, selector: ForgeSelector | None = None
) -> dict | None:
    """Sync the shape ledger from the bound repos and read the accounting.

    Since #2788 this is the ledger's sync point, not just a measurement:
    every walk upserts the extracted shapes (new → unclassified, vanished →
    stamped), re-stamps snippet reference locations as canonical, and then
    reports the ACCOUNTING — how many shapes carry a classification at all —
    rather than the old "has a snippet" fraction. Unclassified is the todo
    (note 2786).

    None means "nothing to measure" — the owner's keyring serves none of the
    project's bound repos (#2778). That is the ordinary state for a
    forge-less user and every caller treats it as silence, not failure; the
    ledger is untouched in that case. Forge errors (unreachable, bad token)
    RAISE — the two callers are a refresh button and a background task, and
    both want to know.

    ``user_id`` is the project OWNER's id: the cache lives there, and the
    keyring resolved here must be the same one every other read uses.
    """
    from scribe.services import shape_ledger
    from scribe.services.forge import ForgeError

    if selector is None:
        selector = await get_forges(user_id, project_id)
    if not selector.configured:
        return None

    served: list[tuple[str, str]] = []
    recorded = await _recorded_locations(user_id, project_id)
    # The proposer's canon catalog, read once per refresh and shared across
    # the project's repos (#2792).
    canons = None
    proposer_stats = {"examined": 0, "proposed": 0, "semantic_checked": 0}
    for key in await keys_for_project(user_id, project_id):
        hit = selector.resolve(key)
        if hit is None:
            continue  # bound to a host no connection serves
        forge, api_repo = hit
        ref = await forge.default_branch(api_repo)
        definitions = definitions_from_archive(await forge.archive(api_repo, ref))
        # The head commit is provenance sugar on the ledger rows; failing to
        # learn it must not fail the sync — the ref names the point well
        # enough and the row timestamps carry the when.
        try:
            marker = await forge.latest_commit(api_repo, "", ref) or ref
        except ForgeError:
            marker = ref
        await shape_ledger.sync_repo_shapes(
            project_id, key, definitions, seen_marker=marker
        )
        served.append((key, ref))
        # Propose while the bodies are in hand — the one moment they exist.
        # Canonical marking below only touches rows the proposer leaves
        # alone (a canon's own location never gets a proposal), so the order
        # is immaterial; the proposer must not be able to fail the refresh.
        try:
            if canons is None:
                canons = await shape_ledger.canon_catalog(user_id)
            stats = await shape_ledger.propose_for_repo(
                user_id, project_id, key, definitions, canons=canons
            )
            for k in proposer_stats:
                proposer_stats[k] += stats.get(k, 0)
        except Exception:
            logger.warning("shape proposer failed for %s", key, exc_info=True)
    if not served:
        return None

    await shape_ledger.mark_canonicals(project_id, recorded)
    try:
        await shape_ledger.apply_derive_groups(project_id)
    except Exception:
        logger.warning("derive-first grouping failed", exc_info=True)
    # The button-B pass (#2793): shapes new since the PREVIOUS computation,
    # where a canon dominates. The previous computation's stamp is the cache;
    # a first seed has none, so it flags nothing (everything is new then).
    try:
        previous = await get_setting(user_id, f"{_CACHE_KEY_PREFIX}{project_id}")
        since = None
        if previous:
            stamp = (json.loads(previous) or {}).get("computed_at")
            since = datetime.fromisoformat(stamp) if stamp else None
        await shape_ledger.flag_divergence(project_id, since=since)
    except Exception:
        logger.warning("divergence pass failed", exc_info=True)

    # Project-wide readout, deliberately wider than this walk: a second bound
    # repo that was unreachable today still has live rows, and they count.
    rows = await shape_ledger.live_rows(project_id)
    counts = {"canonical": 0, "instance": 0, "variant": 0, "exempt": 0,
              "unclassified": 0}
    for row in rows:
        counts[row.status] = counts.get(row.status, 0) + 1
    by_repo: dict[str, dict[str, int]] = {}
    for row in rows:
        agg = by_repo.setdefault(row.repo_key, {"total": 0, "accounted": 0})
        agg["total"] += 1
        agg["accounted"] += row.status != "unclassified"

    unclassified = counts.pop("unclassified")
    proposals = shape_ledger.proposal_summary(rows)
    divergence = shape_ledger.divergence_summary(rows)
    return {
        "total": len(rows),
        "accounted": len(rows) - unclassified,
        "unclassified": unclassified,
        "counts": counts,
        # The proposer's standing (#2792): canon proposals awaiting a
        # confirm, the largest derive-first groups, and what this refresh did.
        "proposed": proposals["proposed"],
        "derive_groups": proposals["derive_groups"],
        "proposer": proposer_stats,
        # The divergence readout (#2793): button B where button A is canon,
        # and judged shapes whose bodies moved since they were judged.
        "divergent": divergence["divergent"],
        "divergence": divergence["divergence"],
        "recheck": divergence["recheck"],
        # Honesty flag, not decoration: every surface that shows the number
        # is expected to carry it through.
        "estimate": True,
        "computed_at": datetime.now(timezone.utc).isoformat(),
        "repos": [
            {"repo": key, "ref": ref,
             **by_repo.get(key, {"total": 0, "accounted": 0})}
            for key, ref in served
        ],
        "largest_gaps": largest_gaps([
            (r.path, r.kind, r.symbol, r.status != "unclassified")
            for r in rows
        ]),
    }


async def refresh_coverage(
    user_id: int, project_id: int, *, selector: ForgeSelector | None = None
) -> dict | None:
    """Compute and cache. The only writer of the cache key."""
    coverage = await compute_coverage(user_id, project_id, selector=selector)
    if coverage is not None:
        await set_setting(
            user_id, f"{_CACHE_KEY_PREFIX}{project_id}", json.dumps(coverage)
        )
    return coverage


# The arrival-moment self-seed (#2802). A ledger that has never been computed
# used to wait for someone to find the UI Refresh button; entering the project
# is the moment the number is wanted, so entering is the moment it seeds.
_SEED_MAX_AGE = timedelta(hours=24)

# Projects with a refresh already running — concurrent enters must not fetch
# the same tarball N times. In-process on purpose: the cost being bounded is
# per-process forge traffic, and a rare double-fetch across workers is
# harmless (the upsert is idempotent).
_inflight_seed: set[int] = set()


async def refresh_if_stale(
    user_id: int, project_id: int, cached: dict | None = None
) -> None:
    """Background-refresh a project's ledger when its readout is absent or
    older than a day. Fire-and-forget material (background.spawn): every exit
    is quiet, every failure a WARNING — a seed must never surface as an
    enter_project error. ``user_id`` is the project OWNER (whose keyring and
    cache this is); pass ``cached`` when the caller already read it.
    """
    try:
        if cached is None:
            cached = await cached_coverage(user_id, project_id)
        if cached:
            try:
                computed = datetime.fromisoformat(cached.get("computed_at") or "")
                if datetime.now(timezone.utc) - computed < _SEED_MAX_AGE:
                    return
            except ValueError:
                pass  # an unreadable stamp reads as stale
        if project_id in _inflight_seed:
            return
        selector = await get_forges(user_id, project_id)
        if not selector.configured:
            return  # rule #115: forge-less stays exactly as it was
        _inflight_seed.add(project_id)
        try:
            await refresh_coverage(user_id, project_id, selector=selector)
        finally:
            _inflight_seed.discard(project_id)
    except Exception:
        logger.warning(
            "background coverage seed failed for project %s", project_id,
            exc_info=True,
        )


async def refresh_for_caller(caller_id: int, project_id: int) -> dict:
    """The explicit agent-facing refresh (#2802) — synchronous, named errors.

    Write-gated: recomputing spends the owner's forge API budget and rewrites
    ledger rows' seen-markers, so a read share doesn't grant it. Resolution
    runs on the OWNER's keyring like every other forge read. Raises
    ValueError with a fixable message instead of silently measuring nothing —
    the caller is an agent mid-task, and "None" would strand it exactly the
    way the button-only path did.
    """
    from scribe.models import async_session
    from scribe.models.project import Project
    from scribe.services import access

    if not await access.can_write_project(caller_id, project_id):
        raise ValueError(f"project {project_id} not found or no write access")
    async with async_session() as session:
        project = await session.get(Project, project_id)
    if project is None:
        raise ValueError(f"project {project_id} not found")
    owner_id = project.user_id or caller_id
    selector = await get_forges(owner_id, project_id)
    if not selector.configured:
        raise ValueError(
            "No forge connection serves this project — the owner adds one "
            "under Settings → Integrations → Git Forges"
        )
    coverage = await refresh_coverage(owner_id, project_id, selector=selector)
    if coverage is None:
        raise ValueError(
            "No bound repo is served by the owner's forge connections — "
            "bind_repo the project's repo on a host a connection serves"
        )
    return coverage


async def cached_coverage(user_id: int, project_id: int) -> dict | None:
    """The last computed summary, or None — never computes."""
    raw = await get_setting(user_id, f"{_CACHE_KEY_PREFIX}{project_id}", "")
    if not raw:
        return None
    try:
        parsed = json.loads(raw)
    except ValueError:
        return None
    return parsed if isinstance(parsed, dict) else None


def coverage_line(coverage: dict) -> str:
    """The one-line evidence-carrying summary enter_project surfaces."""
    day = (coverage.get("computed_at") or "")[:10]
    counts = coverage.get("counts") or {}
    breakdown = " · ".join(
        f"{counts[k]} {k}"
        for k in ("canonical", "instance", "variant", "exempt")
        if counts.get(k)
    )
    line = (
        f"shape accounting: {coverage.get('accounted', 0)}"
        f"/{coverage.get('total', 0)} shapes accounted for"
    )
    if breakdown:
        line += f" — {breakdown}"
    line += f" (estimate{', computed ' + day if day else ''})"
    unclassified = coverage.get("unclassified", 0)
    if unclassified:
        line += f"; {unclassified} unclassified"
        standing = []
        if coverage.get("proposed"):
            standing.append(f"{coverage['proposed']} proposed")
        n_groups = len(coverage.get("derive_groups") or [])
        if n_groups:
            standing.append(f"{n_groups} derive group{'s' if n_groups != 1 else ''}")
        if coverage.get("divergent"):
            standing.append(f"{coverage['divergent']} DIVERGENT")
        if standing:
            line += f" ({', '.join(standing)})"
        gaps = [g["dir"] for g in coverage.get("largest_gaps") or []]
        if gaps:
            line += ", largest: " + ", ".join(gaps)
    if coverage.get("recheck"):
        line += f"; {coverage['recheck']} judged shape{'s' if coverage['recheck'] != 1 else ''} changed since judged — recheck"
    return line