Files
FabledScribe/tests/test_integration_rule_moment_judgments.py
T
bvandeusenandClaude Opus 5.5 4e1320120d
CI & Build / Python lint (push) Successful in 2s
CI & Build / Plugin hooks (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 54s
CI & Build / integration (push) Successful in 1m19s
CI & Build / Python tests (push) Successful in 2m3s
CI & Build / Build & push image (push) Successful in 18s
feat(moments): a mount that keeps arriving where it does not apply proposes its own removal (milestone 458 step 7b, #4955)
The open-after-moment signal proposes a mount; nothing proposed taking one
off, so a wrong mount was noise at every occurrence until someone happened
to notice. rule_misfired(rule_id, moment, why, reached_by) records a report
against a MOUNTED pair, counted per distinct day (the MCP door carries no
session id) on a new rule_moment_judgments.misfire column (migration 0119,
backup v24). At three days the response carries a line asking the agent to
offer the operator the fix - reject takes the rule off, unmap_action stops
the action reaching the moment, confirm keeps the mount and stops the
asking - and Settings > Moments lists it as an unmount proposal with the
reasons and the actions that reached it. A re-mount clears the count.

Taught in moments.md, missed-retrieval.md and the reply hold's wording.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 18:19:59 -04:00

333 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Real-Postgres tests for moment proposals and judgments (milestone 458 step 7).
What the step promises, against the real tables:
- the pass reads only rules nobody has answered for, and any answer — a
proposal, a mount, "no moment fits" — takes a rule off that list;
- a proposal mounts nothing; a confirm mounts, a reject is kept, and a judged
pair is never proposed again by either source;
- an edit that takes a moment off a rule is a rejection the signal respects;
- the signal counts SESSIONS, not opens, and asks once the bar is crossed;
- only the owner can propose, judge, or feed the signal;
- the misfire (step 7b): only a mount can misfire, reports count DAYS, the
bar makes an unmount proposal, reject takes it off, confirm keeps it and
stops the asking, and a re-mount starts the count again.
"""
import pytest
import pytest_asyncio
from sqlalchemy import select
from scribe.models import async_session
from scribe.models.rule_moment_judgment import RuleMomentJudgment
from scribe.models.rulebook import Rulebook
from scribe.models.user import User
from scribe.services import rule_moment_judgments as judgments_svc
from scribe.services import rulebooks as rulebooks_svc
from tests.helpers import ensure_user
pytestmark = [pytest.mark.integration, pytest.mark.usefixtures("_dispose_engine")]
OWNER_USERNAME = "moment_judgments_owner"
STRANGER_USERNAME = "moment_judgments_stranger"
async def _purge_books(username: str) -> None:
"""At SETUP, for the reason test_integration_rule_moments gives."""
async with async_session() as s:
for user in (await s.execute(
select(User).where(User.username == username)
)).scalars().all():
for book in (await s.execute(
select(Rulebook).where(Rulebook.owner_user_id == user.id)
)).scalars().all():
await s.delete(book)
await s.commit()
@pytest_asyncio.fixture
async def world():
for name in (OWNER_USERNAME, STRANGER_USERNAME):
await _purge_books(name)
async with async_session() as s:
owner = await ensure_user(s, OWNER_USERNAME)
stranger = await ensure_user(s, STRANGER_USERNAME)
await s.commit()
uid, sid = owner.id, stranger.id
book = await rulebooks_svc.create_rulebook(uid, "Judgment fixtures")
topic = await rulebooks_svc.create_topic(book.id, uid, "verification")
done = await rulebooks_svc.create_rule(
topic.id, uid, "Done means delivered and checked",
"Work is done when it is delivered and its automated checks pass.",
when_to_apply="about to tell the operator a piece of work is finished",
)
style = await rulebooks_svc.create_rule(
topic.id, uid, "Dates are written day first",
"Every date shown to a person is written day, month, year.",
when_to_apply="formatting a date for display",
)
return {"uid": uid, "sid": sid, "done": done, "style": style}
async def _row(rule_id: int, moment: str) -> RuleMomentJudgment | None:
async with async_session() as s:
return (await s.execute(
select(RuleMomentJudgment).where(
RuleMomentJudgment.rule_id == rule_id, RuleMomentJudgment.moment == moment,
)
)).scalar_one_or_none()
async def _unjudged_ids(uid: int) -> set[int]:
return {r["id"] for r in (await judgments_svc.unjudged_rules(uid, limit=100))["rules"]}
async def test_any_answer_takes_a_rule_off_the_pass(world):
uid, done, style = world["uid"], world["done"], world["style"]
assert {done.id, style.id} <= await _unjudged_ids(uid)
out = await judgments_svc.propose(uid, [
{"rule_id": done.id, "moments": ["work.finish", "reply.report"], "why": "about finishing"},
{"rule_id": style.id, "none": "about how a date looks, whenever one is shown"},
])
assert out["proposed"] == 2 and out["no_moment"] == 1 and not out["refused"]
assert not ({done.id, style.id} & await _unjudged_ids(uid))
# A proposal mounted nothing.
assert (await rulebooks_svc.list_rule_moments([done.id])).get(done.id) is None
async def test_a_confirm_mounts_and_a_reject_is_never_proposed_again(world):
uid, done = world["uid"], world["done"]
await judgments_svc.propose(uid, [
{"rule_id": done.id, "moments": ["work.finish", "work.plan"], "why": "w"},
])
pending = await judgments_svc.pending(uid, rule_id=done.id)
assert [p["moment"] for p in pending["rules"][0]["proposals"]] == ["work.finish", "work.plan"]
out = await judgments_svc.judge(uid, [
{"rule_id": done.id, "moment": "work.finish", "verdict": "confirm", "note": "yes"},
{"rule_id": done.id, "moment": "work.plan", "verdict": "reject", "note": "too early"},
])
assert not out["refused"]
assert (await rulebooks_svc.list_rule_moments([done.id]))[done.id] == ["work.finish"]
confirmed = await _row(done.id, "work.finish")
assert (confirmed.state, confirmed.source, confirmed.note) == ("confirmed", "pass", "yes")
assert (await _row(done.id, "work.plan")).state == "rejected"
assert (await judgments_svc.pending(uid))["total"] == 0
again = await judgments_svc.propose(uid, [
{"rule_id": done.id, "moments": ["work.plan", "work.finish"], "why": "w"},
])
assert again["proposed"] == 0
assert {s["moment"] for s in again["skipped"]} == {"work.plan", "work.finish"}
async def test_an_edit_that_unmounts_is_a_rejection_the_signal_respects(world):
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.deliver"])
await rulebooks_svc.set_rule_moments(done.id, uid, [])
row = await _row(done.id, "work.deliver")
assert (row.state, row.source) == ("rejected", "edit")
for session in ("a", "b", "c", "d"):
assert await judgments_svc.co_occurred(
uid, done.id, ["work.deliver"], situation=session,
) == ""
assert (await _row(done.id, "work.deliver")).evidence is None
async def test_the_signal_counts_sessions_and_asks_once_the_bar_is_crossed(world):
uid, done = world["uid"], world["done"]
# Ubiquitous moments are not evidence; a repeat in one session counts once.
assert await judgments_svc.co_occurred(uid, done.id, ["work.run", "work.change"],
situation="s1") == ""
assert await _row(done.id, "work.run") is None
for situation in ("s1", "s1", "s2"):
assert await judgments_svc.co_occurred(uid, done.id, ["work.verify"],
situation=situation) == ""
line = await judgments_svc.co_occurred(uid, done.id, ["work.verify"], situation="s3")
assert f"#{done.id}" in line and "`work.verify`" in line and "3 distinct sessions" in line
assert "judge_rule_moments" in line
row = await _row(done.id, "work.verify")
assert (row.state, row.source) == ("suggested", "signal")
# Asked once; the cooldown keeps the next open quiet.
assert await judgments_svc.co_occurred(uid, done.id, ["work.verify"], situation="s4") == ""
# And it waits for the operator like any proposal.
pending = await judgments_svc.pending(uid, rule_id=done.id)
[proposal] = pending["rules"][0]["proposals"]
assert proposal["source"] == "signal" and proposal["evidence"]["situations"] == 4
async def test_a_mount_overturns_no_moment_fits(world):
uid, style = world["uid"], world["style"]
await judgments_svc.propose(uid, [{"rule_id": style.id, "none": "about how a date looks"}])
assert (await _row(style.id, "")).state == "confirmed"
await judgments_svc.judge(uid, [
{"rule_id": style.id, "moment": "reply.report", "verdict": "confirm", "note": "dates in reports"},
])
assert (await _row(style.id, "")).state == "rejected"
async def test_only_the_owner_can_propose_judge_or_feed_the_signal(world):
sid, done = world["sid"], world["done"]
out = await judgments_svc.propose(sid, [{"rule_id": done.id, "moments": ["work.finish"]}])
assert out["proposed"] == 0 and out["refused"]
judged = await judgments_svc.judge(sid, [
{"rule_id": done.id, "moment": "work.finish", "verdict": "confirm"},
])
assert judged["refused"] and not judged["judged"]
for situation in ("a", "b", "c"):
assert await judgments_svc.co_occurred(sid, done.id, ["work.verify"],
situation=situation) == ""
assert await _row(done.id, "work.verify") is None
assert done.id not in await _unjudged_ids(sid)
assert (await judgments_svc.pending(sid))["total"] == 0
async def test_the_open_resolves_acts_through_the_installs_mappings(world):
"""`opened_after` reads the acts as delivery does: a push is a deliver."""
uid, done = world["uid"], world["done"]
acts = [
{"tool_name": "Bash", "tool_input": {"command": "git push origin dev"}},
{"tool_name": "Bash", "tool_input": {"command": "ls"}},
"not an act",
]
out = await judgments_svc.opened_after(uid, done.id, acts, session_id="s1")
assert "work.deliver" in out["moments"]
assert "work.run" not in out["moments"]
assert (await _row(done.id, "work.deliver")).source == "signal"
# ── The misfire (step 7b) ────────────────────────────────────────────────────
@pytest.fixture
def days(monkeypatch):
"""Each report lands on the day named next, so a test can cross days
without waiting for them. `misfire_situation` is the one place a day
becomes a situation, so this is what a real date change does."""
queue: list[str] = []
def situation(_now):
from scribe.services import lesson_rules
return lesson_rules.situation_key("d", queue.pop(0))
monkeypatch.setattr(judgments_svc, "misfire_situation", situation)
return queue
async def _misfire(uid, rule_id, moment="work.finish", why="closing a docs-only task",
reached_by="status=done"):
return await judgments_svc.misfired(uid, rule_id, moment, why=why, reached_by=reached_by)
async def test_only_a_mount_can_misfire_and_only_with_a_reason(world, days):
uid, sid, done = world["uid"], world["sid"], world["done"]
out = await _misfire(uid, done.id)
assert not out["recorded"] and "not mounted on work.finish" in out["error"]
assert "when_to_apply" in out["error"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
out = await judgments_svc.misfired(uid, done.id, "work.finish", why=" ")
assert not out["recorded"] and "why" in out["error"]
out = await judgments_svc.misfired(uid, done.id, "work.finished", why="x")
assert not out["recorded"] and "work.finished" in out["error"]
out = await _misfire(sid, done.id)
assert out == {"recorded": False, "error": "not a rule you own"}
assert (await _row(done.id, "work.finish")).misfire is None
async def test_misfires_count_days_and_propose_the_unmount(world, days):
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
days.extend(["2026-10-01", "2026-10-01", "2026-10-02"])
for _ in range(3):
out = await _misfire(uid, done.id)
assert out["recorded"] and out["context"] == ""
assert out["days"] == 2 and out["bar"] == 3
assert (await judgments_svc.pending(uid, rule_id=done.id))["rules"] == []
days.append("2026-10-03")
out = await _misfire(uid, done.id, why="closing a planning stub",
reached_by="status=cancelled")
line = out["context"]
assert out["days"] == 3
assert f"#{done.id}" in line and "`work.finish`" in line and "3 distinct days" in line
assert "`status=done` ×3" in line and "`status=cancelled` ×1" in line
assert '"verdict": "reject"' in line and "unmap_action" in line and '"confirm"' in line
# Asked once; the cooldown keeps the next report quiet.
days.append("2026-10-04")
assert (await _misfire(uid, done.id))["context"] == ""
[entry] = (await judgments_svc.pending(uid, rule_id=done.id))["rules"]
assert entry["mounted"] == ["work.finish"]
[proposal] = entry["proposals"]
assert (proposal["proposal"], proposal["source"], proposal["moment"]) == (
"unmount", "misfire", "work.finish")
assert proposal["evidence"]["situations"] == 4
assert proposal["reached_by"] == {"status=done": 4, "status=cancelled": 1}
assert len(proposal["reasons"]) == judgments_svc.MISFIRE_REASONS
assert proposal["why"] == "closing a docs-only task"
async def _past_the_bar(uid, rule_id, days, moment="work.finish"):
days.extend(["2026-10-01", "2026-10-02", "2026-10-03"])
for _ in range(3):
out = await _misfire(uid, rule_id, moment=moment)
assert out["context"]
async def test_reject_takes_the_mount_off(world, days):
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish", "reply.report"])
await _past_the_bar(uid, done.id, days)
out = await judgments_svc.judge(uid, [
{"rule_id": done.id, "moment": "work.finish", "verdict": "reject", "note": "not at task close"},
])
assert out["refused"] == []
assert (await rulebooks_svc.list_rule_moments([done.id]))[done.id] == ["reply.report"]
assert (await _row(done.id, "work.finish")).state == "rejected"
assert (await judgments_svc.pending(uid, rule_id=done.id))["rules"] == []
# And it cannot misfire where it no longer is.
assert not (await _misfire(uid, done.id))["recorded"]
async def test_confirm_keeps_the_mount_and_stops_the_asking(world, days):
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
await _past_the_bar(uid, done.id, days)
await judgments_svc.judge(uid, [
{"rule_id": done.id, "moment": "work.finish", "verdict": "confirm", "note": "it does belong"},
])
assert (await rulebooks_svc.list_rule_moments([done.id]))[done.id] == ["work.finish"]
row = await _row(done.id, "work.finish")
assert row.state == "confirmed" and row.misfire["kept_note"] == "it does belong"
assert (await judgments_svc.pending(uid, rule_id=done.id))["rules"] == []
# Reports still count, and ask nothing — past the cooldown too.
days.extend(["2026-10-10", "2026-10-11"])
for _ in range(2):
out = await _misfire(uid, done.id)
assert out["recorded"] and out["context"] == ""
assert out["kept"]["note"] == "it does belong"
async def test_a_remount_starts_the_count_again(world, days):
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
await _past_the_bar(uid, done.id, days)
await rulebooks_svc.set_rule_moments(done.id, uid, [])
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
row = await _row(done.id, "work.finish")
assert row.state == "confirmed" and row.misfire is None
async def test_a_mount_with_no_judgment_row_gets_one_on_its_first_misfire(world, days):
"""Mounts made before migration 0118 have no row; a misfire must still count."""
uid, done = world["uid"], world["done"]
await rulebooks_svc.set_rule_moments(done.id, uid, ["work.finish"])
async with async_session() as s:
await s.delete(await s.get(RuleMomentJudgment, (await _row(done.id, "work.finish")).id))
await s.commit()
days.append("2026-10-01")
assert (await _misfire(uid, done.id))["recorded"]
row = await _row(done.id, "work.finish")
assert (row.state, row.source) == ("confirmed", "edit")
assert len(row.misfire["situations"]) == 1