"""The review surface: rows whose evidence no longer meets the bar, and canons whose own rows no longer agree what they are (#4204, #4208). WHAT THESE GUARD, and it is a property rather than an output shape. The ledger was poisoned by a machine writing permanent classifications that nobody read — 32 rows on one project, 334 on another. The correction is NOT a machine that un-writes them; that is the same mistake pointed the other way. It is a reader that puts the evidence in front of a judge. So the tests below assert two things hardest: * a judgment is NEVER listed as weak, however old and whatever its score would have been — an agent's decision is a different kind of evidence, not a worse one; * the listing changes nothing, which is checked by giving it rows and asserting the rows come back untouched. The round-trip test exists because the score used to live only inside a prose sentence. Writer and parser are now one constant apart, and a change to one that the other does not follow is exactly the kind of silent rot the hook's own history is made of. """ import pytest from scribe.services.shape_ledger import ( _RESEMBLE_MIN, _RESEMBLE_REASON, _REVIEW_ROWS_SHOWN, canon_coherence, stamp_score, stamps_to_review, ) class _Row: """A ledger row, as the review reader touches one.""" def __init__(self, *, path="src/a.py", symbol="f", kind="sym", status="instance", snippet_id=1, reason=None, classified_by="hook", signature="async def f():", vanished_at=None, classified_at=None): self.path, self.symbol, self.kind = path, symbol, kind self.status, self.snippet_id, self.reason = status, snippet_id, reason self.classified_by, self.signature = classified_by, signature self.vanished_at, self.classified_at = vanished_at, classified_at def _stamped(score, **kw): return _Row(reason=_RESEMBLE_REASON.format(sid=1, score=score), **kw) # ── the writer and the parser are one constant apart ────────────────────── @pytest.mark.parametrize("score", [0.0, 0.5, 0.69, 0.77, 0.8, 0.95, 1.0]) def test_every_score_the_hook_can_write_reads_back(score) -> None: written = _RESEMBLE_REASON.format(sid=4204, score=score) assert stamp_score(written) == pytest.approx(round(score, 2)) def test_the_real_reasons_from_the_poisoned_ledgers_parse() -> None: """Verbatim from Portal and Scribe rows, so the parser is pinned to text that actually exists in the database rather than to text it generates.""" assert stamp_score("hook: pulled #3283; payload resembles it (0.69)") == 0.69 assert stamp_score("hook: pulled #3283; payload resembles it (0.77)") == 0.77 assert stamp_score("hook: pulled #2860; payload resembles it (0.68)") == 0.68 @pytest.mark.parametrize("reason", [ None, "", " ", "hook: pulled #3461; payload references `icon-btn`", # the OTHER evidence "an agent wrote prose here", "hook: pulled #1; payload resembles it", # no score ]) def test_anything_that_is_not_a_score_reads_as_no_score(reason) -> None: """None means "not written on a score", never "scored zero" — the two must not collapse, because one is a judgment and the other is weak.""" assert stamp_score(reason) is None # ── weak: only hook stamps, only below the floor ────────────────────────── def test_a_stamp_below_the_floor_is_listed_with_its_evidence() -> None: out = stamps_to_review([_stamped(0.69)]) assert out["weak_count"] == 1 row = out["weak"][0] assert row["score"] == 0.69 assert row["form"] == "async-fn" assert row["signature"] == "async def f():" assert out["floor"] == _RESEMBLE_MIN def test_a_stamp_at_or_above_the_floor_is_not_listed() -> None: assert stamps_to_review([_stamped(_RESEMBLE_MIN)])["weak_count"] == 0 assert stamps_to_review([_stamped(0.95)])["weak_count"] == 0 @pytest.mark.parametrize("by", ["agent", "audit", "import", "mechanical"]) def test_a_judgment_is_never_weak_however_it_would_have_scored(by) -> None: """The measured case: 302 of Scribe's 334 rows under one canon were `audit`-classified with no score at all. Listing those as weak would invite an agent to withdraw the only judgments in the ledger that were ever actually made by one.""" rows = [_Row(classified_by=by, reason=None), _Row(classified_by=by, reason=_RESEMBLE_REASON.format(sid=1, score=0.10))] assert stamps_to_review(rows)["weak_count"] == 0 def test_an_unclassified_or_vanished_row_is_not_listed() -> None: assert stamps_to_review([_stamped(0.1, status="unclassified")])["weak_count"] == 0 assert stamps_to_review([_stamped(0.1, vanished_at="gone")])["weak_count"] == 0 def test_the_weakest_evidence_is_listed_first() -> None: out = stamps_to_review([ _stamped(0.77, symbol="c"), _stamped(0.68, symbol="a"), _stamped(0.71, symbol="b"), ]) assert [r["symbol"] for r in out["weak"]] == ["a", "b", "c"] def test_top_bounds_the_listing_but_not_the_count() -> None: out = stamps_to_review([_stamped(0.7, symbol=f"s{i}") for i in range(9)], top=3) assert out["weak_count"] == 9 and len(out["weak"]) == 3 # ── incoherent: a canon whose own members disagree ──────────────────────── def _member(sig, sid=7, **kw): return _Row(snippet_id=sid, signature=sig, classified_by="audit", **kw) def test_a_canon_whose_members_agree_is_not_listed() -> None: rows = [_member("async def a():"), _member("async def b():"), _member("async def c():"), _member("async def d():")] assert stamps_to_review(rows)["incoherent_count"] == 0 def test_a_canon_whose_members_are_all_different_things_is_listed() -> None: """Portal's #3283, in miniature: a class, a getter, a test and a binding recorded as instances of one pattern — and in four different files, as they really were. The paths matter now: a callable sharing a file with a class is read as that class's method (#4220), so putting them all in one file would test the excuse rather than the disagreement.""" rows = [_member("class SessionAbsent(RuntimeError):", path="src/errors.py", symbol="SessionAbsent"), _member("def build_channel() -> str:", path="src/channel.py", symbol="build_channel"), _member("async def attach(self) -> None:", path="src/attach.py", symbol="attach"), _member("MAX = 10", path="src/limits.py", symbol="MAX")] out = stamps_to_review(rows) assert out["incoherent_count"] == 1 entry = out["incoherent"][0] assert entry["snippet_id"] == 7 and entry["judged"] == 4 assert set(entry["forms"]) == {"type", "fn", "async-fn", "binding"} # The two callables carry the vote; the class and the binding are what # the canon cannot account for, and they are what the reader is shown. assert entry["family"] == "callable" assert {s["symbol"] for s in entry["strangers"]} == {"SessionAbsent", "MAX"} def test_too_few_readable_members_says_nothing_either_way() -> None: """Two rows that disagree is not an incoherent canon, it is two rows. The floor is the same one the density check uses.""" rows = [_member("class A:"), _member("def b():")] assert stamps_to_review(rows)["incoherent_count"] == 0 def test_unreadable_signatures_do_not_manufacture_incoherence() -> None: rows = [_member(""), _member(" "), _member("???"), _member("")] assert stamps_to_review(rows)["incoherent_count"] == 0 def test_a_canon_carrying_weak_stamps_is_ranked_above_one_that_is_not() -> None: clean = [_member("class A:", sid=1), _member("def b():", sid=1), _member("MAX = 1", sid=1), _member("async def c():", sid=1)] dirty = [_Row(snippet_id=2, signature=s, classified_by="hook", reason=_RESEMBLE_REASON.format(sid=2, score=0.70)) for s in ("class D:", "def e():", "MAX = 2", "async def f():")] out = stamps_to_review(clean + dirty) assert out["incoherent_count"] == 2 assert out["incoherent"][0]["snippet_id"] == 2 assert out["incoherent"][0]["weak_rows"] == 4 assert out["incoherent"][1]["weak_rows"] == 0 # ── the property that matters most: it is a reader ──────────────────────── def test_the_review_changes_nothing_about_the_rows_it_reads() -> None: """Asserted rather than assumed. The whole design claim of this surface is that it reports and never acts; a future refactor that "helpfully" resets a status here would pass every other test in this file.""" rows = [_stamped(0.69), _member("class A:"), _member("def b():"), _member("MAX = 1"), _member("async def c():")] before = [(r.status, r.snippet_id, r.classified_by, r.reason) for r in rows] stamps_to_review(rows) after = [(r.status, r.snippet_id, r.classified_by, r.reason) for r in rows] assert before == after def test_the_service_carries_no_machinery_for_bulk_withdrawal() -> None: """A structural guard (rule 167) on the design decision, not on output. If someone adds an auto-retire path to the ledger, this fails and the conversation about whether a machine may un-judge in bulk happens again — deliberately, rather than in a diff nobody read.""" import scribe.services.shape_ledger as ledger body = open(ledger.__file__).read() for banned in ("def retire_weak", "def auto_unclassify", "def bulk_withdraw"): assert banned not in body, banned # ── coherence is about what a canon CANNOT account for (#4220) ──────────── # # The first version of this check asked whether every member shared a form. # On its first live day the only two canons it reported were both sound, # while the genuinely polluted one had already been cleaned by hand — a # surface whose whole output is noise is one that stops being read. # # The obvious repair — group by family, keep the majority test — is a trap, # and `test_family_grouping_alone_would_have_gone_blind` is the guard that # stops anyone walking back into it. def _model(sym, path): """A SQLAlchemy model class, as #2844 holds one.""" return _member(f"class {sym}(Base, TimestampMixin):", path=path) def _to_dict(path): """The serialiser that model is required to carry, beside it.""" return _member("def to_dict(self) -> dict:", path=path, symbol="to_dict") def test_a_class_and_the_to_dict_beside_it_are_one_shape() -> None: """#2844 exactly: the SQLAlchemy model convention, whose own text is about the class AND the to_dict it must carry. Live it scored 37/62 = 0.597 and was called incoherent for containing precisely what it says it contains.""" rows = [] for i in range(6): rows += [_model(f"M{i}", f"src/scribe/models/m{i}.py"), _to_dict(f"src/scribe/models/m{i}.py")] out = stamps_to_review(rows) assert out["incoherent_count"] == 0 def test_a_sync_starter_and_the_async_tick_it_schedules_are_one_shape() -> None: """#2849: four loop-starters and three ticks. `shape_family` already collapses fn and async-fn for the divergence gate; this surface was the one place still asking at form level.""" rows = [_member("def start_notification_loop() -> None:", path="src/n.py"), _member("def start_log_retention_loop() -> None:", path="src/l.py"), _member("def start_auth_token_retention_loop() -> None:", path="src/a.py"), _member("async def _notification_tick() -> None:", path="src/n.py"), _member("async def _retention_tick() -> None:", path="src/l.py"), _member("async def _auth_token_retention_tick() -> None:", path="src/a.py")] assert stamps_to_review(rows)["incoherent_count"] == 0 def test_family_grouping_alone_would_have_gone_blind() -> None: """THE REGRESSION GUARD. Polluted #2844 as it stood this morning: 37 model classes, 25 to_dict methods beside them, and 31 rows that had no business being there. Counted as families that is 56 callables to 37 types — 0.602, a clean pass under any majority test. Those 31 rows are the ones the surface exists to find, so a change that lets this canon read as coherent has broken the feature while keeping every other test in this file green.""" rows = [] for i in range(37): rows.append(_model(f"M{i}", f"src/scribe/models/m{i}.py")) for i in range(25): rows.append(_to_dict(f"src/scribe/models/m{i % 37}.py")) strangers = (["src/scribe/routes/rulebooks.py"] * 2 + ["src/scribe/services/embeddings.py"] * 7 + ["frontend/src/stores/rulebooks.ts"] * 3 + ["tests/test_services_rulebooks.py"] * 19) for i, path in enumerate(strangers): rows.append(_member(f"async def f{i}():", path=path, symbol=f"f{i}")) out = stamps_to_review(rows) assert out["incoherent_count"] == 1, "the polluted canon must still be caught" entry = out["incoherent"][0] assert entry["families"]["callable"] > entry["families"]["type"] assert entry["family"] == "type", "methods must not outvote their classes" assert entry["attached"] == 25 assert entry["stranger_count"] == len(strangers) == 31 # The listing is capped, the count is not — a reader must be able to # tell twelve strangers from thirty-one. assert len(entry["strangers"]) == _REVIEW_ROWS_SHOWN assert all(not s["path"].startswith("src/scribe/models") for s in entry["strangers"]) def test_a_callable_in_a_file_with_no_class_is_a_stranger() -> None: """The excuse is "method of a member", not "callable anywhere". Without this, any function in the repo would be excused by the existence of a class somewhere else in the canon.""" rows = [_model("A", "src/models/a.py"), _to_dict("src/models/a.py"), _model("B", "src/models/b.py"), _to_dict("src/models/b.py"), _member("def helper():", path="src/services/free.py", symbol="helper"), _member("def other():", path="src/services/free.py", symbol="other")] out = stamps_to_review(rows) assert out["incoherent_count"] == 1 assert {s["symbol"] for s in out["incoherent"][0]["strangers"]} == {"helper", "other"} def test_the_strangers_are_named_not_a_sample_of_the_majority() -> None: """The reader's question is which rows are wrong. A sample of the agreeing majority cannot answer it, which is what the first version returned.""" # Seven and two: 2 of 9 is over `_STRANGER_SHARE`, 2 of 11 is under it. # The tolerance is real and the counts here sit deliberately on the far # side of it — see the test below for the near side. rows = [_member("async def a():", path=f"src/ok{i}.py", symbol=f"a{i}") for i in range(7)] rows.append(_member("class Odd:", path="src/odd.py", symbol="Odd")) rows.append(_member("class Odder:", path="src/odder.py", symbol="Odder")) out = stamps_to_review(rows) assert out["incoherent_count"] == 1 assert {s["symbol"] for s in out["incoherent"][0]["strangers"]} == {"Odd", "Odder"} def test_a_single_odd_row_in_a_large_canon_is_tolerated() -> None: """One stranger in twenty is a row to fix, not a canon that has stopped meaning anything. The surface is for the second thing.""" rows = [_member("async def a():", path=f"src/ok{i}.py", symbol=f"a{i}") for i in range(19)] rows.append(_member("class Odd:", path="src/odd.py", symbol="Odd")) assert stamps_to_review(rows)["incoherent_count"] == 0 def test_an_incoherence_made_of_judged_rows_says_so() -> None: """The discriminator that tells a too-strict check from a bad ledger. Both canons flagged on the first live day were entirely audit-judged, and a reader could not see that without opening each one.""" judged = [_member("async def a():", path=f"src/j{i}.py", symbol=f"a{i}") for i in range(4)] judged += [_member("class J:", path="src/jc.py", symbol="J"), _member("class J2:", path="src/jc2.py", symbol="J2")] out = stamps_to_review(judged) assert out["incoherent_count"] == 1 assert out["incoherent"][0]["unattended"] == 0 assert out["incoherent"][0]["weak_rows"] == 0 def test_the_coherence_verdict_reads_rows_and_writes_nothing() -> None: """`canon_coherence` is called on live ORM rows; it must not touch them.""" rows = [_model("A", "src/models/a.py"), _to_dict("src/models/a.py"), _member("def loose():", path="src/x.py", symbol="loose")] before = [(r.path, r.symbol, r.status, r.snippet_id, r.signature) for r in rows] canon_coherence(rows) assert [(r.path, r.symbol, r.status, r.snippet_id, r.signature) for r in rows] == before