FabledScribe/tests/test_integration_snippet_locations.py

"""Real-Postgres integration test for the snippet location reverse lookup (#2083).

Runs only in the CI integration lane (real Postgres, schema built by
`alembic upgrade head`, which includes migration 0070's `notes.data` + GIN index).
This exercises what the unit mocks cannot:

  - the jsonpath `@?` containment behind `_location_clause` — including
    `starts with` for a path prefix, and the fact that parts must match within a
    SINGLE `locations` entry;
  - that the SQL dialect and the Python dialect (`location_matches`, used by the
    semantic arm) return the SAME rows. Those two must never drift, or a snippet
    is findable by wording and invisible by location;
  - `backfill_snippet_data`, which is what stops a pre-0070 snippet from being
    silently reported as "nothing recorded here."
"""
from unittest.mock import AsyncMock, patch

import pytest
import pytest_asyncio
from sqlalchemy import delete, func, select

from scribe.models import async_session, engine
from scribe.models.note import Note
from scribe.models.user import User
from scribe.services.knowledge import location_matches, location_parts
from scribe.services.snippets import (
    SNIPPET_NOTE_TYPE,
    backfill_snippet_data,
    compose_body,
    compose_data,
    list_snippets,
    snippet_fields,
)

pytestmark = pytest.mark.integration


@pytest_asyncio.fixture(autouse=True)
async def _dispose_engine():
    # Per-loop pool: dispose after each test (see test_integration_db_maintenance).
    yield
    await engine.dispose()


def _loc(repo="", path="", symbol=""):
    return {"repo": repo, "path": path, "symbol": symbol}


@pytest_asyncio.fixture
async def seeded():
    """A user with four snippets covering the cases the filter has to separate.

    Returns (user_id, {label: note_id}).
    """
    async with async_session() as s:
        user = User(username="snippet_loc_itest")
        s.add(user)
        await s.flush()

        def _snippet(name, locations, code="return 1"):
            return Note(
                user_id=user.id,
                note_type=SNIPPET_NOTE_TYPE,
                title=f"{name} — does a thing",
                body=compose_body(code=code, language="python", locations=locations),
                tags=["python", "snippet"],
                data=compose_data(name=name, language="python", locations=locations),
            )

        nested = _snippet("nested", [_loc("Scribe", "frontend/src/lib/x.ts", "helper")])
        sibling = _snippet("sibling", [_loc("Scribe", "frontend/srcmap.ts", "other")])
        # Two locations, deliberately crossed: repo Scribe at src/a.py and repo
        # Portal at src/b.py. repo=Scribe + path=src/b.py must NOT match it.
        multi = _snippet(
            "multi",
            [_loc("Scribe", "src/a.py", "alpha"), _loc("Portal", "src/b.py", "beta")],
        )
        # No structured location at all — must never satisfy a location filter,
        # and must not error the query either.
        placeless = _snippet("placeless", [])
        # A plain note (not a snippet) with no `data` — proves the jsonpath
        # operator tolerates NULL rather than raising.
        plain = Note(user_id=user.id, title="plain", body="no data here")

        s.add_all([nested, sibling, multi, placeless, plain])
        await s.commit()
        ids = (
            user.id,
            {
                "nested": nested.id,
                "sibling": sibling.id,
                "multi": multi.id,
                "placeless": placeless.id,
                "plain": plain.id,
            },
        )
    yield ids
    user_id = ids[0]
    async with async_session() as s:
        await s.execute(delete(Note).where(Note.user_id == user_id))
        await s.execute(delete(User).where(User.id == user_id))
        await s.commit()


async def _ids_for(user_id: int, **parts) -> set[int]:
    items, total = await list_snippets(user_id, **parts)
    assert total == len(items), "total must match the filtered page, not the whole list"
    return {i["id"] for i in items}


@pytest.mark.asyncio
async def test_repo_filter_finds_every_snippet_recorded_in_it(seeded):
    user_id, ids = seeded
    found = await _ids_for(user_id, repo="Scribe")
    assert found == {ids["nested"], ids["sibling"], ids["multi"]}
    assert ids["placeless"] not in found


@pytest.mark.asyncio
async def test_exact_path_finds_only_that_file(seeded):
    user_id, ids = seeded
    assert await _ids_for(user_id, path="frontend/src/lib/x.ts") == {ids["nested"]}


@pytest.mark.asyncio
async def test_path_prefix_reaches_into_subdirectories(seeded):
    user_id, ids = seeded
    # ...and stops at the directory boundary: srcmap.ts is a sibling, not a child.
    assert await _ids_for(user_id, path="frontend/src") == {ids["nested"]}
    assert await _ids_for(user_id, path="frontend/src/") == {ids["nested"]}
    assert await _ids_for(user_id, path="frontend") == {ids["nested"], ids["sibling"]}


@pytest.mark.asyncio
async def test_parts_must_match_within_one_location(seeded):
    user_id, ids = seeded
    assert await _ids_for(user_id, repo="Scribe", path="src/a.py") == {ids["multi"]}
    # Same snippet, but repo and path belong to two different entries.
    assert await _ids_for(user_id, repo="Scribe", path="src/b.py") == set()
    assert await _ids_for(user_id, repo="Portal", path="src/b.py") == {ids["multi"]}


@pytest.mark.asyncio
async def test_symbol_filter_and_an_unrecorded_location(seeded):
    user_id, ids = seeded
    assert await _ids_for(user_id, symbol="helper") == {ids["nested"]}
    assert await _ids_for(user_id, repo="NoSuchRepo") == set()


@pytest.mark.asyncio
async def test_no_location_filter_still_lists_everything(seeded):
    """The filter is opt-in — a plain browse must not lose the placeless snippet."""
    found = await _ids_for(user_id=seeded[0])
    ids = seeded[1]
    assert {ids["nested"], ids["sibling"], ids["multi"], ids["placeless"]} <= found
    assert ids["plain"] not in found  # not a snippet


@pytest.mark.asyncio
async def test_keyword_arm_applies_the_location_filter(seeded):
    """Search AND place compose. The embedder is stubbed out so this isolates the
    keyword (ILIKE) arm, which is the second place the SQL clause has to appear."""
    user_id, ids = seeded
    with patch(
        "scribe.services.embeddings.semantic_search_notes",
        AsyncMock(return_value=[]),
    ):
        assert await _ids_for(user_id, q="nested", repo="Scribe") == {ids["nested"]}
        # Matches the query but not the place.
        assert await _ids_for(user_id, q="nested", repo="Portal") == set()


@pytest.mark.asyncio
async def test_semantic_arm_applies_the_location_filter(seeded):
    """The semantic arm holds its candidates in memory, so it filters in Python.
    A query that matches no title or body leaves only that arm in play."""
    user_id, ids = seeded
    async with async_session() as s:
        rows = list(
            (
                await s.execute(
                    select(Note)
                    .where(Note.user_id == user_id)
                    .where(Note.note_type == SNIPPET_NOTE_TYPE)
                )
            )
            .scalars()
            .all()
        )
    candidates = [(0.9, n) for n in rows]

    with patch(
        "scribe.services.embeddings.semantic_search_notes",
        AsyncMock(return_value=candidates),
    ):
        # No keyword can match this, so every hit below came from the stub.
        assert await _ids_for(user_id, q="zzzznomatch") == {
            ids["nested"], ids["sibling"], ids["multi"], ids["placeless"],
        }
        assert await _ids_for(user_id, q="zzzznomatch", path="frontend/src") == {
            ids["nested"],
        }
        assert await _ids_for(user_id, q="zzzznomatch", repo="Scribe", path="src/b.py") == set()


@pytest.mark.asyncio
async def test_sql_and_python_dialects_agree_on_the_same_rows(seeded):
    """The drift guard: whatever the SQL arm returns, the Python arm must too."""
    user_id, _ids = seeded
    async with async_session() as s:
        rows = list(
            (
                await s.execute(
                    select(Note)
                    .where(Note.user_id == user_id)
                    .where(Note.note_type == SNIPPET_NOTE_TYPE)
                )
            )
            .scalars()
            .all()
        )
        by_id = {n.id: n.data for n in rows}

    cases = [
        {"repo": "Scribe"},
        {"path": "frontend/src"},
        {"path": "frontend/src/lib/x.ts"},
        {"repo": "Scribe", "path": "src/a.py"},
        {"repo": "Scribe", "path": "src/b.py"},
        {"symbol": "beta"},
        {"repo": "Scribe", "path": "frontend", "symbol": "helper"},
    ]
    for case in cases:
        sql_ids = await _ids_for(user_id, **case)
        parts = location_parts(**case)
        python_ids = {nid for nid, data in by_id.items() if location_matches(data, parts)}
        assert sql_ids == python_ids, f"dialects disagree on {case}"


@pytest.mark.asyncio
async def test_backfill_makes_a_pre_0070_snippet_findable_by_location(seeded):
    """A snippet stored before migration 0070 has no `data`, so the location query
    can't see it. Unfilled, it reads as "nothing recorded here" — the failure the
    backfill exists to prevent.

    `data` is OMITTED from the constructor on purpose: that leaves a genuine SQL
    NULL, which is the shape migration 0070 actually left behind. Passing
    `data=None` would store the JSON encoding of null instead (SQLAlchemy JSON
    defaults to none_as_null=False) — a different state, covered by the next test.
    """
    user_id, _ids = seeded
    locations = [_loc("Legacy", "old/path/y.py", "legacy_helper")]
    async with async_session() as s:
        old = Note(
            user_id=user_id,
            note_type=SNIPPET_NOTE_TYPE,
            title="legacy — recorded before the data column existed",
            body=compose_body(code="return 2", language="python", locations=locations),
            tags=["python", "snippet"],
        )
        s.add(old)
        await s.commit()
        old_id = old.id

    assert await _ids_for(user_id, repo="Legacy") == set()

    filled = await backfill_snippet_data()
    assert filled >= 1

    assert await _ids_for(user_id, repo="Legacy") == {old_id}
    assert await _ids_for(user_id, path="old/path") == {old_id}

    # The mirror agrees with the body it was derived from, and the body is untouched.
    async with async_session() as s:
        row = (await s.execute(select(Note).where(Note.id == old_id))).scalar_one()
        assert row.data["locations"] == locations
        assert snippet_fields(row)["symbol"] == "legacy_helper"
        assert "old/path/y.py" in row.body


@pytest.mark.asyncio
async def test_backfill_fills_a_json_null_and_is_idempotent(seeded):
    """The OTHER unfilled shape: `data=None` persists JSON `null`, not SQL NULL
    (SQLAlchemy JSON defaults to none_as_null=False), so an `IS NULL` test alone
    would walk past this row. Also pins idempotence — a fieldless snippet settles
    at `{}` instead of staying unfilled and being rescanned on every boot."""
    user_id, _ids = seeded
    async with async_session() as s:
        bare = Note(
            user_id=user_id,
            note_type=SNIPPET_NOTE_TYPE,
            title="",
            body="",
            tags=[],
            data=None,
        )
        s.add(bare)
        await s.commit()
        bare_id = bare.id

    # Confirm the premise rather than assuming it: this row really is JSON null,
    # so the assertion below is testing the second arm of the predicate.
    async with async_session() as s:
        kind = (
            await s.execute(
                select(func.jsonb_typeof(Note.data)).where(Note.id == bare_id)
            )
        ).scalar_one()
        assert kind == "null"

    assert await backfill_snippet_data() >= 1
    assert await backfill_snippet_data() == 0

    async with async_session() as s:
        row = (await s.execute(select(Note).where(Note.id == bare_id))).scalar_one()
        assert row.data == {}