FabledScribe/tests/test_stamps_to_review.py

"""The review surface: rows whose evidence no longer meets the bar, and
canons whose own rows no longer agree what they are (#4204, #4208).

WHAT THESE GUARD, and it is a property rather than an output shape. The
ledger was poisoned by a machine writing permanent classifications that
nobody read — 32 rows on one project, 334 on another. The correction is NOT
a machine that un-writes them; that is the same mistake pointed the other
way. It is a reader that puts the evidence in front of a judge.

So the tests below assert two things hardest:

  * a judgment is NEVER listed as weak, however old and whatever its score
    would have been — an agent's decision is a different kind of evidence,
    not a worse one;
  * the listing changes nothing, which is checked by giving it rows and
    asserting the rows come back untouched.

The round-trip test exists because the score used to live only inside a
prose sentence. Writer and parser are now one constant apart, and a change
to one that the other does not follow is exactly the kind of silent rot the
hook's own history is made of.
"""


import pytest

from scribe.services.shape_ledger import (
    _RESEMBLE_MIN, _RESEMBLE_REASON, _REVIEW_ROWS_SHOWN, canon_coherence,
    stamp_score, stamps_to_review,
)


class _Row:
    """A ledger row, as the review reader touches one."""

    def __init__(self, *, path="src/a.py", symbol="f", kind="sym",
                 status="instance", snippet_id=1, reason=None,
                 classified_by="hook", signature="async def f():",
                 vanished_at=None, classified_at=None):
        self.path, self.symbol, self.kind = path, symbol, kind
        self.status, self.snippet_id, self.reason = status, snippet_id, reason
        self.classified_by, self.signature = classified_by, signature
        self.vanished_at, self.classified_at = vanished_at, classified_at


def _stamped(score, **kw):
    return _Row(reason=_RESEMBLE_REASON.format(sid=1, score=score), **kw)


# ── the writer and the parser are one constant apart ──────────────────────

@pytest.mark.parametrize("score", [0.0, 0.5, 0.69, 0.77, 0.8, 0.95, 1.0])
def test_every_score_the_hook_can_write_reads_back(score) -> None:
    written = _RESEMBLE_REASON.format(sid=4204, score=score)
    assert stamp_score(written) == pytest.approx(round(score, 2))


def test_the_real_reasons_from_the_poisoned_ledgers_parse() -> None:
    """Verbatim from Portal and Scribe rows, so the parser is pinned to text
    that actually exists in the database rather than to text it generates."""
    assert stamp_score("hook: pulled #3283; payload resembles it (0.69)") == 0.69
    assert stamp_score("hook: pulled #3283; payload resembles it (0.77)") == 0.77
    assert stamp_score("hook: pulled #2860; payload resembles it (0.68)") == 0.68


@pytest.mark.parametrize("reason", [
    None, "", "   ",
    "hook: pulled #3461; payload references `icon-btn`",   # the OTHER evidence
    "an agent wrote prose here",
    "hook: pulled #1; payload resembles it",               # no score
])
def test_anything_that_is_not_a_score_reads_as_no_score(reason) -> None:
    """None means "not written on a score", never "scored zero" — the two
    must not collapse, because one is a judgment and the other is weak."""
    assert stamp_score(reason) is None


# ── weak: only hook stamps, only below the floor ──────────────────────────

def test_a_stamp_below_the_floor_is_listed_with_its_evidence() -> None:
    out = stamps_to_review([_stamped(0.69)])
    assert out["weak_count"] == 1
    row = out["weak"][0]
    assert row["score"] == 0.69
    assert row["form"] == "async-fn"
    assert row["signature"] == "async def f():"
    assert out["floor"] == _RESEMBLE_MIN


def test_a_stamp_at_or_above_the_floor_is_not_listed() -> None:
    assert stamps_to_review([_stamped(_RESEMBLE_MIN)])["weak_count"] == 0
    assert stamps_to_review([_stamped(0.95)])["weak_count"] == 0


@pytest.mark.parametrize("by", ["agent", "audit", "import", "mechanical"])
def test_a_judgment_is_never_weak_however_it_would_have_scored(by) -> None:
    """The measured case: 302 of Scribe's 334 rows under one canon were
    `audit`-classified with no score at all. Listing those as weak would
    invite an agent to withdraw the only judgments in the ledger that were
    ever actually made by one."""
    rows = [_Row(classified_by=by, reason=None),
            _Row(classified_by=by,
                 reason=_RESEMBLE_REASON.format(sid=1, score=0.10))]
    assert stamps_to_review(rows)["weak_count"] == 0


def test_an_unclassified_or_vanished_row_is_not_listed() -> None:
    assert stamps_to_review([_stamped(0.1, status="unclassified")])["weak_count"] == 0
    assert stamps_to_review([_stamped(0.1, vanished_at="gone")])["weak_count"] == 0


def test_the_weakest_evidence_is_listed_first() -> None:
    out = stamps_to_review([
        _stamped(0.77, symbol="c"), _stamped(0.68, symbol="a"),
        _stamped(0.71, symbol="b"),
    ])
    assert [r["symbol"] for r in out["weak"]] == ["a", "b", "c"]


def test_top_bounds_the_listing_but_not_the_count() -> None:
    out = stamps_to_review([_stamped(0.7, symbol=f"s{i}") for i in range(9)], top=3)
    assert out["weak_count"] == 9 and len(out["weak"]) == 3


# ── incoherent: a canon whose own members disagree ────────────────────────

def _member(sig, sid=7, **kw):
    return _Row(snippet_id=sid, signature=sig, classified_by="audit", **kw)


def test_a_canon_whose_members_agree_is_not_listed() -> None:
    rows = [_member("async def a():"), _member("async def b():"),
            _member("async def c():"), _member("async def d():")]
    assert stamps_to_review(rows)["incoherent_count"] == 0


def test_a_canon_whose_members_are_all_different_things_is_listed() -> None:
    """Portal's #3283, in miniature: a class, a getter, a test and a binding
    recorded as instances of one pattern — and in four different files, as
    they really were. The paths matter now: a callable sharing a file with a
    class is read as that class's method (#4220), so putting them all in one
    file would test the excuse rather than the disagreement."""
    rows = [_member("class SessionAbsent(RuntimeError):", path="src/errors.py",
                    symbol="SessionAbsent"),
            _member("def build_channel() -> str:", path="src/channel.py",
                    symbol="build_channel"),
            _member("async def attach(self) -> None:", path="src/attach.py",
                    symbol="attach"),
            _member("MAX = 10", path="src/limits.py", symbol="MAX")]
    out = stamps_to_review(rows)
    assert out["incoherent_count"] == 1
    entry = out["incoherent"][0]
    assert entry["snippet_id"] == 7 and entry["judged"] == 4
    assert set(entry["forms"]) == {"type", "fn", "async-fn", "binding"}
    # The two callables carry the vote; the class and the binding are what
    # the canon cannot account for, and they are what the reader is shown.
    assert entry["family"] == "callable"
    assert {s["symbol"] for s in entry["strangers"]} == {"SessionAbsent", "MAX"}


def test_too_few_readable_members_says_nothing_either_way() -> None:
    """Two rows that disagree is not an incoherent canon, it is two rows.
    The floor is the same one the density check uses."""
    rows = [_member("class A:"), _member("def b():")]
    assert stamps_to_review(rows)["incoherent_count"] == 0


def test_unreadable_signatures_do_not_manufacture_incoherence() -> None:
    rows = [_member(""), _member("   "), _member("???"), _member("")]
    assert stamps_to_review(rows)["incoherent_count"] == 0


def test_a_canon_carrying_weak_stamps_is_ranked_above_one_that_is_not() -> None:
    clean = [_member("class A:", sid=1), _member("def b():", sid=1),
             _member("MAX = 1", sid=1), _member("async def c():", sid=1)]
    dirty = [_Row(snippet_id=2, signature=s, classified_by="hook",
                  reason=_RESEMBLE_REASON.format(sid=2, score=0.70))
             for s in ("class D:", "def e():", "MAX = 2", "async def f():")]
    out = stamps_to_review(clean + dirty)
    assert out["incoherent_count"] == 2
    assert out["incoherent"][0]["snippet_id"] == 2
    assert out["incoherent"][0]["weak_rows"] == 4
    assert out["incoherent"][1]["weak_rows"] == 0


# ── the property that matters most: it is a reader ────────────────────────

def test_the_review_changes_nothing_about_the_rows_it_reads() -> None:
    """Asserted rather than assumed. The whole design claim of this surface
    is that it reports and never acts; a future refactor that "helpfully"
    resets a status here would pass every other test in this file."""
    rows = [_stamped(0.69), _member("class A:"), _member("def b():"),
            _member("MAX = 1"), _member("async def c():")]
    before = [(r.status, r.snippet_id, r.classified_by, r.reason) for r in rows]
    stamps_to_review(rows)
    after = [(r.status, r.snippet_id, r.classified_by, r.reason) for r in rows]
    assert before == after


def test_the_service_carries_no_machinery_for_bulk_withdrawal() -> None:
    """A structural guard (rule 167) on the design decision, not on output.
    If someone adds an auto-retire path to the ledger, this fails and the
    conversation about whether a machine may un-judge in bulk happens again
    — deliberately, rather than in a diff nobody read."""
    import scribe.services.shape_ledger as ledger

    body = open(ledger.__file__).read()
    for banned in ("def retire_weak", "def auto_unclassify", "def bulk_withdraw"):
        assert banned not in body, banned


# ── coherence is about what a canon CANNOT account for (#4220) ────────────
#
# The first version of this check asked whether every member shared a form.
# On its first live day the only two canons it reported were both sound,
# while the genuinely polluted one had already been cleaned by hand — a
# surface whose whole output is noise is one that stops being read.
#
# The obvious repair — group by family, keep the majority test — is a trap,
# and `test_family_grouping_alone_would_have_gone_blind` is the guard that
# stops anyone walking back into it.

def _model(sym, path):
    """A SQLAlchemy model class, as #2844 holds one."""
    return _member(f"class {sym}(Base, TimestampMixin):", path=path)


def _to_dict(path):
    """The serialiser that model is required to carry, beside it."""
    return _member("def to_dict(self) -> dict:", path=path, symbol="to_dict")


def test_a_class_and_the_to_dict_beside_it_are_one_shape() -> None:
    """#2844 exactly: the SQLAlchemy model convention, whose own text is
    about the class AND the to_dict it must carry. Live it scored
    37/62 = 0.597 and was called incoherent for containing precisely what it
    says it contains."""
    rows = []
    for i in range(6):
        rows += [_model(f"M{i}", f"src/scribe/models/m{i}.py"),
                 _to_dict(f"src/scribe/models/m{i}.py")]
    out = stamps_to_review(rows)
    assert out["incoherent_count"] == 0


def test_a_sync_starter_and_the_async_tick_it_schedules_are_one_shape() -> None:
    """#2849: four loop-starters and three ticks. `shape_family` already
    collapses fn and async-fn for the divergence gate; this surface was the
    one place still asking at form level."""
    rows = [_member("def start_notification_loop() -> None:", path="src/n.py"),
            _member("def start_log_retention_loop() -> None:", path="src/l.py"),
            _member("def start_auth_token_retention_loop() -> None:", path="src/a.py"),
            _member("async def _notification_tick() -> None:", path="src/n.py"),
            _member("async def _retention_tick() -> None:", path="src/l.py"),
            _member("async def _auth_token_retention_tick() -> None:", path="src/a.py")]
    assert stamps_to_review(rows)["incoherent_count"] == 0


def test_family_grouping_alone_would_have_gone_blind() -> None:
    """THE REGRESSION GUARD. Polluted #2844 as it stood this morning: 37
    model classes, 25 to_dict methods beside them, and 31 rows that had no
    business being there. Counted as families that is 56 callables to 37
    types — 0.602, a clean pass under any majority test. Those 31 rows are
    the ones the surface exists to find, so a change that lets this canon
    read as coherent has broken the feature while keeping every other test
    in this file green."""
    rows = []
    for i in range(37):
        rows.append(_model(f"M{i}", f"src/scribe/models/m{i}.py"))
    for i in range(25):
        rows.append(_to_dict(f"src/scribe/models/m{i % 37}.py"))
    strangers = (["src/scribe/routes/rulebooks.py"] * 2
                 + ["src/scribe/services/embeddings.py"] * 7
                 + ["frontend/src/stores/rulebooks.ts"] * 3
                 + ["tests/test_services_rulebooks.py"] * 19)
    for i, path in enumerate(strangers):
        rows.append(_member(f"async def f{i}():", path=path, symbol=f"f{i}"))

    out = stamps_to_review(rows)
    assert out["incoherent_count"] == 1, "the polluted canon must still be caught"
    entry = out["incoherent"][0]
    assert entry["families"]["callable"] > entry["families"]["type"]
    assert entry["family"] == "type", "methods must not outvote their classes"
    assert entry["attached"] == 25
    assert entry["stranger_count"] == len(strangers) == 31
    # The listing is capped, the count is not — a reader must be able to
    # tell twelve strangers from thirty-one.
    assert len(entry["strangers"]) == _REVIEW_ROWS_SHOWN
    assert all(not s["path"].startswith("src/scribe/models")
               for s in entry["strangers"])


def test_a_callable_in_a_file_with_no_class_is_a_stranger() -> None:
    """The excuse is "method of a member", not "callable anywhere". Without
    this, any function in the repo would be excused by the existence of a
    class somewhere else in the canon."""
    rows = [_model("A", "src/models/a.py"), _to_dict("src/models/a.py"),
            _model("B", "src/models/b.py"), _to_dict("src/models/b.py"),
            _member("def helper():", path="src/services/free.py", symbol="helper"),
            _member("def other():", path="src/services/free.py", symbol="other")]
    out = stamps_to_review(rows)
    assert out["incoherent_count"] == 1
    assert {s["symbol"] for s in out["incoherent"][0]["strangers"]} == {"helper", "other"}


def test_the_strangers_are_named_not_a_sample_of_the_majority() -> None:
    """The reader's question is which rows are wrong. A sample of the
    agreeing majority cannot answer it, which is what the first version
    returned."""
    # Seven and two: 2 of 9 is over `_STRANGER_SHARE`, 2 of 11 is under it.
    # The tolerance is real and the counts here sit deliberately on the far
    # side of it — see the test below for the near side.
    rows = [_member("async def a():", path=f"src/ok{i}.py", symbol=f"a{i}")
            for i in range(7)]
    rows.append(_member("class Odd:", path="src/odd.py", symbol="Odd"))
    rows.append(_member("class Odder:", path="src/odder.py", symbol="Odder"))
    out = stamps_to_review(rows)
    assert out["incoherent_count"] == 1
    assert {s["symbol"] for s in out["incoherent"][0]["strangers"]} == {"Odd", "Odder"}


def test_a_single_odd_row_in_a_large_canon_is_tolerated() -> None:
    """One stranger in twenty is a row to fix, not a canon that has stopped
    meaning anything. The surface is for the second thing."""
    rows = [_member("async def a():", path=f"src/ok{i}.py", symbol=f"a{i}")
            for i in range(19)]
    rows.append(_member("class Odd:", path="src/odd.py", symbol="Odd"))
    assert stamps_to_review(rows)["incoherent_count"] == 0


def test_an_incoherence_made_of_judged_rows_says_so() -> None:
    """The discriminator that tells a too-strict check from a bad ledger.
    Both canons flagged on the first live day were entirely audit-judged,
    and a reader could not see that without opening each one."""
    judged = [_member("async def a():", path=f"src/j{i}.py", symbol=f"a{i}")
              for i in range(4)]
    judged += [_member("class J:", path="src/jc.py", symbol="J"),
               _member("class J2:", path="src/jc2.py", symbol="J2")]
    out = stamps_to_review(judged)
    assert out["incoherent_count"] == 1
    assert out["incoherent"][0]["unattended"] == 0
    assert out["incoherent"][0]["weak_rows"] == 0


def test_the_coherence_verdict_reads_rows_and_writes_nothing() -> None:
    """`canon_coherence` is called on live ORM rows; it must not touch them."""
    rows = [_model("A", "src/models/a.py"), _to_dict("src/models/a.py"),
            _member("def loose():", path="src/x.py", symbol="loose")]
    before = [(r.path, r.symbol, r.status, r.snippet_id, r.signature) for r in rows]
    canon_coherence(rows)
    assert [(r.path, r.symbol, r.status, r.snippet_id, r.signature)
            for r in rows] == before


# ── a weak stamp is listed for judgment and counted by nobody (#4608) ─────
#
# The floor guarded new writes; the rows written before it went on electing
# canons. On two projects a YAML CI snippet "dominated" a Svelte directory on
# the strength of 0.68-0.72 stamps, and every write there was told it
# diverged from it. The readers below must agree with the review surface
# about what "weak" means — one predicate, so they cannot drift apart.

def test_one_predicate_says_what_weak_is() -> None:
    from scribe.services.shape_ledger import is_weak_stamp

    assert is_weak_stamp(_stamped(0.72))
    assert not is_weak_stamp(_stamped(_RESEMBLE_MIN))
    # A by-name stamp carries no score: it is not resemblance evidence at all.
    assert not is_weak_stamp(_Row(reason="hook: pulled #1; payload references `f`"))
    for by in ("agent", "audit", "mechanical"):
        assert not is_weak_stamp(_stamped(0.5, classified_by=by)), by


def test_weak_stamps_do_not_elect_a_dominant_canon() -> None:
    from scribe.services.shape_ledger import dominant_canon

    weak = [_stamped(0.72, path=f"web/src/lib/{i}.ts", symbol=f"f{i}") for i in range(5)]
    assert dominant_canon(weak) is None
    # The same rows judged by an agent do — the kind of evidence is the point.
    judged = [_stamped(0.72, path=f"web/src/lib/{i}.ts", symbol=f"f{i}", classified_by="agent")
              for i in range(5)]
    assert dominant_canon(judged) == (1, 5, 5)
    # And weak rows do not dilute a real majority either: they are not counted
    # as judged at all.
    mixed = judged[:3] + [_stamped(0.7, snippet_id=2, symbol=f"g{i}") for i in range(4)]
    assert dominant_canon(mixed) == (1, 3, 3)


def test_weak_stamps_do_not_say_what_form_a_canon_is() -> None:
    from scribe.services.shape_ledger import FORM_UNKNOWN, canon_form

    weak = [_stamped(0.7, symbol=f"T{i}", signature=f"class T{i}:") for i in range(5)]
    assert canon_form(weak, 1) == FORM_UNKNOWN
    judged = [_stamped(0.7, symbol=f"T{i}", signature=f"class T{i}:", classified_by="audit")
              for i in range(5)]
    assert canon_form(judged + weak, 1) == canon_form(judged, 1) != FORM_UNKNOWN


def test_counting_them_out_leaves_them_listed() -> None:
    """Not counted is not withdrawn: the review still lists every one."""
    rows = [_stamped(0.7, symbol=f"f{i}") for i in range(3)]
    assert stamps_to_review(rows)["weak_count"] == 3