Files
FabledScribe/tests/test_integration_shape_classify.py
T
bvandeusenandClaude Fable 5 ba0030e51d
CI & Build / Python lint (push) Successful in 5s
CI & Build / Plugin hooks (push) Successful in 13s
CI & Build / integration (push) Failing after 34s
CI & Build / TypeScript typecheck (push) Successful in 35s
CI & Build / Python tests (push) Successful in 1m17s
CI & Build / Build & push image (push) Successful in 48s
feat(ledger): mechanical proposer — every refresh proposes instances against canon and groups derive-first candidates; agents confirm in batches (#2792, milestone 294 step 6)
Shapes now carry a content fingerprint (signature + whitespace/comment-
insensitive body_sha; migration 0080) and the proposer runs inside the
coverage refresh, the one moment bodies exist: symbol elsewhere → textual
containment → body references the canon → signature resemblance → semantic
(capped per refresh, unreached rows stay unexamined for the next). A hit is
a proposal on the row (proposed_snippet_id/basis/score), never a
classification; rows with no canon hit group by the derive-first rule
(identical body in ≥2 places, same name in ≥3 files) as proposal_basis=
derive + a group key. list_shapes(proposal=any|canon|derive|<basis>) is the
queue; confirm_shape_proposals(project_id, snippet_id|path|basis) confirms
in batches as agent instances; any classify_shapes/hook stamp retires the
proposal. Readout carries proposed + derive_groups (line, payload, card).
Plugin 0.1.35 (skill: the machine proposes, judgment classifies).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-20 21:30:35 -04:00

496 lines
22 KiB
Python

"""Real-Postgres integration tests for shape classification (#2789).
What mocks can't prove: the all-or-nothing batch against real rows, the
write-ACL gate, the todo query's filters, and the consumer map riding
get_snippet. Ledger rows are seeded through the same sync the coverage walk
uses — no forge needed, the sync takes extracted shapes directly.
"""
import pytest
import pytest_asyncio
from sqlalchemy import select
from scribe.models import async_session, engine
from scribe.models.code_shape import CodeShape
from scribe.models.project import Project
from scribe.models.user import User
from scribe.services.shape_ledger import (
classify_shapes,
list_project_shapes,
snippet_consumers,
sync_repo_shapes,
)
pytestmark = pytest.mark.integration
REPO = "git.example.com/alice/widget"
SHAPES = [
("src/app.py", "sym", "make_app"),
("src/app.py", "sym", "Config"),
("src/util.py", "sym", "helper"),
("web/button.css", "css", "btn"),
]
@pytest_asyncio.fixture(autouse=True)
async def _dispose_engine():
"""Dispose the app's module-level engine after each test.
The engine pools asyncpg connections per event loop, but pytest-asyncio runs
each test on a fresh loop — so without this, test 2 gets handed test 1's
connection bound to a now-dead loop ("Future attached to a different loop").
Disposing in the test's own loop teardown clears the pool cleanly.
"""
yield
await engine.dispose()
async def _user(session, username: str) -> User:
existing = (
await session.execute(select(User).where(User.username == username))
).scalar_one_or_none()
if existing is not None:
return existing
user = User(username=username)
session.add(user)
await session.flush()
return user
@pytest_asyncio.fixture
async def seeded():
"""Owner + outsider, a project with a synced 4-shape ledger, one snippet."""
from scribe.services import snippets as snippets_svc
async with async_session() as s:
owner = await _user(s, "classify_owner")
other = await _user(s, "classify_other")
project = Project(user_id=owner.id, title="Classify target")
s.add(project)
await s.flush()
ids = {"owner": owner.id, "other": other.id, "pid": project.id}
await s.commit()
await sync_repo_shapes(ids["pid"], REPO, SHAPES, seen_marker="main")
snippet = await snippets_svc.create_snippet(
ids["owner"], name="cls_make_app", code="def make_app():\n pass\n",
language="python", repo="Widget", path="src/factory.py",
symbol="factory", project_id=ids["pid"],
)
ids["snippet"] = int(snippet.id)
return ids
@pytest.mark.integration
async def test_classify_applies_judgments_and_reports_unmatched(seeded):
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
out = await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": sid},
{"path": "src/util.py", "symbol": "helper", "status": "exempt",
"reason": "test scaffolding, deliberately local"},
{"path": "web/button.css", "symbol": "btn", "status": "variant",
"snippet_id": sid, "reason": "darker focus ring for the toolbar"},
{"path": "gone.py", "symbol": "nothing", "status": "exempt",
"reason": "x"},
], via="audit")
assert out["classified"] == 3
assert out["unmatched"] == [{"path": "gone.py", "symbol": "nothing"}]
rows, total = await list_project_shapes(owner, pid)
by_symbol = {r.symbol: r for r in rows}
assert total == 4
assert by_symbol["make_app"].status == "instance"
assert by_symbol["make_app"].snippet_id == sid
assert by_symbol["make_app"].classified_by == "audit"
assert by_symbol["helper"].status == "exempt"
assert by_symbol["helper"].reason == "test scaffolding, deliberately local"
assert by_symbol["btn"].status == "variant"
assert by_symbol["Config"].status == "unclassified"
# Withdrawing a judgment returns the shape to the todo, fields cleared.
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "unclassified"},
])
rows, _ = await list_project_shapes(owner, pid, status="unclassified")
assert {r.symbol for r in rows} == {"Config", "make_app"}
make_app = next(r for r in rows if r.symbol == "make_app")
assert make_app.snippet_id is None and make_app.classified_by is None
@pytest.mark.integration
async def test_a_bad_batch_applies_nothing(seeded):
"""All-or-nothing (#2709's lesson): a caller must never learn later that
half a batch silently happened."""
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
with pytest.raises(ValueError) as err:
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": sid},
{"path": "src/util.py", "symbol": "helper", "status": "variant",
"snippet_id": sid}, # variant with no reason: structural error
])
assert "needs a reason" in str(err.value)
rows, _ = await list_project_shapes(owner, pid, status="unclassified")
assert len(rows) == 4 # including make_app — the valid half did NOT apply
# A snippet target the caller can't read is the same: nothing applies.
with pytest.raises(ValueError):
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": 999999999},
])
@pytest.mark.integration
async def test_classification_is_write_gated_and_listing_read_gated(seeded):
other, pid, sid = seeded["other"], seeded["pid"], seeded["snippet"]
with pytest.raises(ValueError):
await classify_shapes(other, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "exempt",
"reason": "not their call to make"},
])
assert await list_project_shapes(other, pid) == ([], 0)
@pytest.mark.integration
async def test_list_filters_compose(seeded):
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": sid},
])
rows, total = await list_project_shapes(owner, pid, path="src")
assert total == 3 and all(r.path.startswith("src/") for r in rows)
# Directory semantics, not string prefix: "sr" matches nothing.
assert (await list_project_shapes(owner, pid, path="sr"))[1] == 0
rows, total = await list_project_shapes(owner, pid, snippet_id=sid)
assert total == 1 and rows[0].symbol == "make_app"
rows, total = await list_project_shapes(
owner, pid, status="unclassified", limit=2
)
assert total == 3 and len(rows) == 2 # paged, with the true total
@pytest.mark.integration
async def test_get_snippet_carries_the_structured_consumer_map(seeded):
from scribe.mcp._context import _user_id_ctx
from scribe.mcp.tools.snippets import get_snippet
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": sid},
{"path": "web/button.css", "symbol": "btn", "status": "variant",
"snippet_id": sid, "reason": "darker focus ring"},
])
token = _user_id_ctx.set(owner)
try:
data = await get_snippet(snippet_id=sid)
finally:
_user_id_ctx.reset(token)
assert [i["path"] for i in data["instances"]] == ["src/app.py"]
assert data["variants"][0]["reason"] == "darker focus ring"
# The map is caller-scoped: an outsider asking the service directly gets
# silence, not another project's file layout.
consumers = await snippet_consumers(seeded["other"], sid)
assert consumers == {"instances": [], "variants": []}
@pytest.mark.integration
async def test_sync_refiles_rows_whose_snippet_was_purged(seeded):
"""The SET NULL companion (#2787): a judgment whose target is hard-deleted
rejoins the todo on the next sync instead of dangling target-less."""
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "instance",
"snippet_id": sid},
])
from scribe.models.note import Note
async with async_session() as s:
note = await s.get(Note, sid)
await s.delete(note) # hard delete, as purge_trash would
await s.commit()
await sync_repo_shapes(pid, REPO, SHAPES, seen_marker="main")
async with async_session() as s:
row = (await s.execute(select(CodeShape).where(
CodeShape.project_id == pid, CodeShape.symbol == "make_app",
))).scalar_one()
assert row.status == "unclassified"
assert row.snippet_id is None
# --- #2791: the write-path feed lands hook evidence as rows -------------------
@pytest.mark.integration
async def test_write_path_stamp_is_evidence_that_yields_to_judgment(seeded):
"""Pulled + referenced → every named shape of the snippet's kind becomes
an instance row, classified_by=hook, carrying the evidence as reason. A
later agent judgment on one of them stands against a re-stamp; the hook
may only overwrite nobody's judgment or its own. The outsider stamps
nothing (write-gated like every other ledger write)."""
from datetime import datetime, timezone
from scribe.services.shape_ledger import stamp_write_path_instances
owner, other, pid, sid = (
seeded["owner"], seeded["other"], seeded["pid"], seeded["snippet"]
)
pulled = {sid: datetime.now(timezone.utc)}
code = "app = factory()\nreturn app\n" # references the snippet's symbol
assert await stamp_write_path_instances(
other, pid, path="src/app.py", shapes=[("sym", "make_app")],
code=code, pulled=pulled,
) == []
stamped = await stamp_write_path_instances(
owner, pid, path="src/app.py",
shapes=[("sym", "make_app"), ("sym", "Config"), ("css", "nope")],
code=code, pulled=pulled,
)
assert {s["symbol"] for s in stamped} == {"make_app", "Config"} # css skipped: no css canon
rows, _ = await list_project_shapes(owner, pid, snippet_id=sid)
by_symbol = {r.symbol: r for r in rows}
assert by_symbol["make_app"].status == "instance"
assert by_symbol["make_app"].classified_by == "hook"
assert by_symbol["make_app"].reason == f"hook: pulled #{sid}; payload references `factory`"
# A judgment lands; the next stamp must leave it alone but may re-stamp
# its own earlier row.
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "exempt",
"reason": "the app factory is its own thing"},
])
again = await stamp_write_path_instances(
owner, pid, path="src/app.py",
shapes=[("sym", "make_app"), ("sym", "Config")], code=code, pulled=pulled,
)
assert {s["symbol"] for s in again} == {"Config"}
rows, _ = await list_project_shapes(owner, pid, path="src/app.py")
by_symbol = {r.symbol: r for r in rows}
assert by_symbol["make_app"].status == "exempt"
assert by_symbol["Config"].status == "instance"
# Neither pulled nor in play → nothing, even with shapes named.
assert await stamp_write_path_instances(
owner, pid, path="src/util.py", shapes=[("sym", "helper")],
code="print('unrelated')", pulled=pulled,
) == []
@pytest.mark.integration
async def test_a_brand_new_shape_gets_a_provisional_row_the_sync_settles(seeded):
"""The shape being written right now has no ledger row yet. With the
hook's repo key it gets a provisional one — seen markers empty — so the
stamp survives until the next sync, which confirms it (sets the marker)
or stamps it vanished. Without a repo key only existing rows are touched."""
from datetime import datetime, timezone
from scribe.services.shape_ledger import stamp_write_path_instances
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
pulled = {sid: datetime.now(timezone.utc)}
code = "def build():\n return factory()\n"
assert await stamp_write_path_instances(
owner, pid, path="src/new.py", shapes=[("sym", "build")],
code=code, pulled=pulled, # no repo_key
) == []
stamped = await stamp_write_path_instances(
owner, pid, path="src/new.py", shapes=[("sym", "build")],
code=code, pulled=pulled, repo_key=REPO,
)
assert [s["symbol"] for s in stamped] == ["build"]
async with async_session() as s:
row = (await s.execute(select(CodeShape).where(
CodeShape.project_id == pid, CodeShape.path == "src/new.py",
))).scalar_one()
assert row.status == "instance" and row.classified_by == "hook"
# "Unset" is the column's empty default — the markers are non-null
# Text, and the sync is what first fills them.
assert row.first_seen_commit == "" and row.last_seen_commit == ""
# The sync sees the shape in the tree → confirmed, stamp intact.
await sync_repo_shapes(
pid, REPO, SHAPES + [("src/new.py", "sym", "build")], seen_marker="abc123",
)
rows, _ = await list_project_shapes(owner, pid, path="src/new.py")
assert rows[0].status == "instance" and rows[0].last_seen_commit == "abc123"
# The sync no longer sees it → vanished, out of the live accounting.
await sync_repo_shapes(pid, REPO, SHAPES, seen_marker="def456")
rows, _ = await list_project_shapes(owner, pid, path="src/new.py")
assert rows == []
rows, _ = await list_project_shapes(owner, pid, path="src/new.py", include_vanished=True)
assert rows[0].vanished_at is not None
@pytest.mark.integration
async def test_recent_pulls_reads_the_usage_stream(seeded):
"""The "actually pulled it" half is the PULLED usage event, inside the
window; a surfacing alone is not a pull."""
from datetime import datetime, timedelta, timezone
from scribe.models.note_usage import PULLED, SURFACED, NoteUsageEvent
from scribe.services.shape_ledger import recent_pulls
owner, sid = seeded["owner"], seeded["snippet"]
now = datetime.now(timezone.utc)
async with async_session() as s:
s.add_all([
NoteUsageEvent(user_id=owner, note_id=sid, event=PULLED, source="mcp_get_snippet"),
NoteUsageEvent(user_id=owner, note_id=sid + 1000, event=SURFACED, source="auto_inject"),
NoteUsageEvent(user_id=owner, note_id=sid + 2000, event=PULLED,
source="mcp_get_snippet", created_at=now - timedelta(days=2)),
])
await s.commit()
pulls = await recent_pulls(owner)
assert sid in pulls
assert sid + 1000 not in pulls
assert sid + 2000 not in pulls
# --- #2792: the mechanical proposer against real rows -------------------------
def _quiet_semantic():
"""The semantic basis needs the embedder; these tests prove the other
bases and the bookkeeping, so it answers "nothing" here."""
from unittest.mock import AsyncMock, patch
from scribe.services import shape_ledger
return patch.object(shape_ledger, "_semantic_canon", AsyncMock(return_value=None))
def _defs(*items):
"""ArchiveShape-like records: (path, kind, name, signature, body_sha, body)."""
import hashlib
out = []
for path, kind, name, signature, body in items:
sha = hashlib.sha1(" ".join(body.split()).encode()).hexdigest()[:16]
out.append((path, kind, name, signature, sha, body))
return out
@pytest.mark.integration
async def test_proposer_proposes_and_confirm_classifies(seeded):
"""Bodies in hand, the proposer records proposals on unclassified rows —
symbol (a second `factory` elsewhere), reference (a call site), and
nothing for the unrelated — skips rows whose content it already judged,
and a scoped confirm turns proposals into agent instances while a
classify on another retires its proposal."""
from scribe.services.shape_ledger import (
confirm_proposals, propose_for_repo,
)
owner, other, pid, sid = (
seeded["owner"], seeded["other"], seeded["pid"], seeded["snippet"]
)
defs = _defs(
("src/app.py", "sym", "make_app", "def make_app():", "def make_app():\n app = factory()\n return app"),
("src/app.py", "sym", "Config", "class Config:", "class Config:\n debug = False"),
("src/util.py", "sym", "helper", "def helper(x):", "def helper(x):\n return x"),
("src/dup.py", "sym", "factory", "def factory():", "def factory():\n return 1"),
("web/button.css", "css", "btn", ".btn {", ".btn {\n color: red;\n}"),
)
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
with _quiet_semantic():
stats = await propose_for_repo(owner, pid, REPO, defs)
assert stats == {"examined": 5, "proposed": 2, "semantic_checked": 2}
rows, total = await list_project_shapes(owner, pid, proposal="canon")
by_symbol = {r.symbol: r for r in rows}
assert total == 2
assert by_symbol["factory"].proposal == {"basis": "symbol", "score": 1.0, "snippet_id": sid}
assert by_symbol["make_app"].proposal == {"basis": "reference", "score": 0.9, "snippet_id": sid}
rows, _ = await list_project_shapes(owner, pid, proposal="reference")
assert [r.symbol for r in rows] == ["make_app"]
# Same content again → nothing re-examined (the semantic cap would
# otherwise be spent on the same rows every refresh). A cap that leaves
# rows unreached leaves them UNexamined, so the next refresh gets them.
with _quiet_semantic():
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 0
await classify_shapes(owner, pid, [
{"path": "src/util.py", "symbol": "helper", "status": "unclassified"},
])
assert (await propose_for_repo(owner, pid, REPO, defs, semantic_cap=0))["semantic_checked"] == 0
assert (await propose_for_repo(owner, pid, REPO, defs))["examined"] == 1
# Outsider can't confirm; the owner confirms by snippet, scoped.
with pytest.raises(ValueError):
await confirm_proposals(other, pid, snippet_id=sid)
assert await confirm_proposals(owner, pid, basis="symbol") == {"confirmed": 1}
rows, _ = await list_project_shapes(owner, pid, snippet_id=sid)
factory = next(r for r in rows if r.symbol == "factory")
assert factory.status == "instance" and factory.classified_by == "agent"
assert factory.reason == "confirmed symbol proposal (1.00)"
assert factory.proposal is None
# A judgment on a proposed row retires the proposal; withdrawing a
# judgment forgets the examination so the next pass proposes afresh.
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "exempt", "reason": "bootstrap"},
])
rows, _ = await list_project_shapes(owner, pid, path="src/app.py")
make_app = next(r for r in rows if r.symbol == "make_app")
assert make_app.status == "exempt" and make_app.proposal is None
await classify_shapes(owner, pid, [
{"path": "src/app.py", "symbol": "make_app", "status": "unclassified"},
])
with _quiet_semantic():
assert (await propose_for_repo(owner, pid, REPO, defs))["proposed"] == 1
rows, _ = await list_project_shapes(owner, pid, proposal="canon")
assert [r.symbol for r in rows] == ["make_app"]
@pytest.mark.integration
async def test_derive_groups_land_on_rows_and_in_the_summary(seeded):
"""Shapes with no canon hit that repeat — identical bodies in two files,
the same name in three — carry a derive proposal, and the readout ranks
the families. A canon proposal keeps a row out of any derive group."""
from scribe.services.shape_ledger import (
apply_derive_groups, live_rows, propose_for_repo, proposal_summary,
)
owner, pid, sid = seeded["owner"], seeded["pid"], seeded["snippet"]
defs = _defs(
("a/one.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
("a/two.py", "sym", "slug", "def slug(t):", "def slug(t):\n return t.lower()"),
("b/x.css", "css", "card", ".card {", ".card { padding: 1px }"),
("b/y.css", "css", "card", ".card {", ".card { padding: 2px }"),
("b/z.css", "css", "card", ".card {", ".card { padding: 3px }"),
("c/only.py", "sym", "alone", "def alone():", "def alone():\n return 0"),
("c/use.py", "sym", "boot", "def boot():", "def boot():\n return factory()"),
)
await sync_repo_shapes(pid, REPO, defs, seen_marker="main")
with _quiet_semantic():
await propose_for_repo(owner, pid, REPO, defs)
assert await apply_derive_groups(pid) == 5
rows, total = await list_project_shapes(owner, pid, proposal="derive")
assert total == 5
groups = {(r.path, r.symbol): r.proposal for r in rows}
assert groups[("a/one.py", "slug")]["group"] == groups[("a/two.py", "slug")]["group"]
assert groups[("a/one.py", "slug")]["group"].startswith("dup:")
assert groups[("b/x.css", "card")] == {"basis": "derive", "score": 3.0, "group": "name:css:card"}
rows, _ = await list_project_shapes(owner, pid, proposal="any")
assert {r.symbol for r in rows} == {"slug", "card", "boot"} # boot: reference proposal
summary = proposal_summary(await live_rows(pid))
assert summary["proposed"] == 1
assert [g["group"] for g in summary["derive_groups"]][0] == "name:css:card"
assert summary["derive_groups"][0]["label"] == ".card"
assert summary["derive_groups"][0]["size"] == 3
# One of the css copies gets judged → the group shrinks on the next pass.
await classify_shapes(owner, pid, [
{"path": "b/z.css", "symbol": "card", "status": "exempt", "reason": "print sheet"},
])
await apply_derive_groups(pid)
rows, _ = await list_project_shapes(owner, pid, proposal="derive")
assert {r.symbol for r in rows} == {"slug"} # 2 files < the name floor